ViewVC Help
View File | Revision Log | Show Annotations | Download File | View Changeset | Root Listing
root/src/trunk/sys/x86/acpica/srat.c
Revision: 12312
Committed: Sat Feb 8 19:34:34 2020 UTC (4 years, 2 months ago) by laffer1
Content type: text/plain
File size: 12854 byte(s)
Log Message:
sync with FreeBSD 11-stable

File Contents

# Content
1 /* $MidnightBSD$ */
2 /*-
3 * Copyright (c) 2010 Hudson River Trading LLC
4 * Written by: John H. Baldwin <jhb@FreeBSD.org>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD: stable/11/sys/x86/acpica/srat.c 322996 2017-08-29 07:01:15Z mav $");
31
32 #include "opt_vm.h"
33
34 #include <sys/param.h>
35 #include <sys/bus.h>
36 #include <sys/kernel.h>
37 #include <sys/lock.h>
38 #include <sys/mutex.h>
39 #include <sys/smp.h>
40 #include <sys/vmmeter.h>
41 #include <vm/vm.h>
42 #include <vm/pmap.h>
43 #include <vm/vm_param.h>
44 #include <vm/vm_page.h>
45 #include <vm/vm_phys.h>
46
47 #include <contrib/dev/acpica/include/acpi.h>
48 #include <contrib/dev/acpica/include/aclocal.h>
49 #include <contrib/dev/acpica/include/actables.h>
50
51 #include <machine/intr_machdep.h>
52 #include <x86/apicvar.h>
53
54 #include <dev/acpica/acpivar.h>
55
56 #if MAXMEMDOM > 1
57 struct cpu_info {
58 int enabled:1;
59 int has_memory:1;
60 int domain;
61 } cpus[MAX_APIC_ID + 1];
62
63 struct mem_affinity mem_info[VM_PHYSSEG_MAX + 1];
64 int num_mem;
65
66 static ACPI_TABLE_SRAT *srat;
67 static vm_paddr_t srat_physaddr;
68
69 static int domain_pxm[MAXMEMDOM];
70 static int ndomain;
71
72 static ACPI_TABLE_SLIT *slit;
73 static vm_paddr_t slit_physaddr;
74 static int vm_locality_table[MAXMEMDOM * MAXMEMDOM];
75
76 static void srat_walk_table(acpi_subtable_handler *handler, void *arg);
77
78 /*
79 * SLIT parsing.
80 */
81
82 static void
83 slit_parse_table(ACPI_TABLE_SLIT *s)
84 {
85 int i, j;
86 int i_domain, j_domain;
87 int offset = 0;
88 uint8_t e;
89
90 /*
91 * This maps the SLIT data into the VM-domain centric view.
92 * There may be sparse entries in the PXM namespace, so
93 * remap them to a VM-domain ID and if it doesn't exist,
94 * skip it.
95 *
96 * It should result in a packed 2d array of VM-domain
97 * locality information entries.
98 */
99
100 if (bootverbose)
101 printf("SLIT.Localities: %d\n", (int) s->LocalityCount);
102 for (i = 0; i < s->LocalityCount; i++) {
103 i_domain = acpi_map_pxm_to_vm_domainid(i);
104 if (i_domain < 0)
105 continue;
106
107 if (bootverbose)
108 printf("%d: ", i);
109 for (j = 0; j < s->LocalityCount; j++) {
110 j_domain = acpi_map_pxm_to_vm_domainid(j);
111 if (j_domain < 0)
112 continue;
113 e = s->Entry[i * s->LocalityCount + j];
114 if (bootverbose)
115 printf("%d ", (int) e);
116 /* 255 == "no locality information" */
117 if (e == 255)
118 vm_locality_table[offset] = -1;
119 else
120 vm_locality_table[offset] = e;
121 offset++;
122 }
123 if (bootverbose)
124 printf("\n");
125 }
126 }
127
128 /*
129 * Look for an ACPI System Locality Distance Information Table ("SLIT")
130 */
131 static int
132 parse_slit(void)
133 {
134
135 if (resource_disabled("slit", 0)) {
136 return (-1);
137 }
138
139 slit_physaddr = acpi_find_table(ACPI_SIG_SLIT);
140 if (slit_physaddr == 0) {
141 return (-1);
142 }
143
144 /*
145 * Make a pass over the table to populate the cpus[] and
146 * mem_info[] tables.
147 */
148 slit = acpi_map_table(slit_physaddr, ACPI_SIG_SLIT);
149 slit_parse_table(slit);
150 acpi_unmap_table(slit);
151 slit = NULL;
152
153 #ifdef VM_NUMA_ALLOC
154 /* Tell the VM about it! */
155 mem_locality = vm_locality_table;
156 #endif
157 return (0);
158 }
159
160 /*
161 * SRAT parsing.
162 */
163
164 /*
165 * Returns true if a memory range overlaps with at least one range in
166 * phys_avail[].
167 */
168 static int
169 overlaps_phys_avail(vm_paddr_t start, vm_paddr_t end)
170 {
171 int i;
172
173 for (i = 0; phys_avail[i] != 0 && phys_avail[i + 1] != 0; i += 2) {
174 if (phys_avail[i + 1] <= start)
175 continue;
176 if (phys_avail[i] < end)
177 return (1);
178 break;
179 }
180 return (0);
181
182 }
183
184 static void
185 srat_parse_entry(ACPI_SUBTABLE_HEADER *entry, void *arg)
186 {
187 ACPI_SRAT_CPU_AFFINITY *cpu;
188 ACPI_SRAT_X2APIC_CPU_AFFINITY *x2apic;
189 ACPI_SRAT_MEM_AFFINITY *mem;
190 int domain, i, slot;
191
192 switch (entry->Type) {
193 case ACPI_SRAT_TYPE_CPU_AFFINITY:
194 cpu = (ACPI_SRAT_CPU_AFFINITY *)entry;
195 domain = cpu->ProximityDomainLo |
196 cpu->ProximityDomainHi[0] << 8 |
197 cpu->ProximityDomainHi[1] << 16 |
198 cpu->ProximityDomainHi[2] << 24;
199 if (bootverbose)
200 printf("SRAT: Found CPU APIC ID %u domain %d: %s\n",
201 cpu->ApicId, domain,
202 (cpu->Flags & ACPI_SRAT_CPU_ENABLED) ?
203 "enabled" : "disabled");
204 if (!(cpu->Flags & ACPI_SRAT_CPU_ENABLED))
205 break;
206 if (cpu->ApicId > MAX_APIC_ID) {
207 printf("SRAT: Ignoring local APIC ID %u (too high)\n",
208 cpu->ApicId);
209 break;
210 }
211
212 if (cpus[cpu->ApicId].enabled) {
213 printf("SRAT: Duplicate local APIC ID %u\n",
214 cpu->ApicId);
215 *(int *)arg = ENXIO;
216 break;
217 }
218 cpus[cpu->ApicId].domain = domain;
219 cpus[cpu->ApicId].enabled = 1;
220 break;
221 case ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY:
222 x2apic = (ACPI_SRAT_X2APIC_CPU_AFFINITY *)entry;
223 if (bootverbose)
224 printf("SRAT: Found CPU APIC ID %u domain %d: %s\n",
225 x2apic->ApicId, x2apic->ProximityDomain,
226 (x2apic->Flags & ACPI_SRAT_CPU_ENABLED) ?
227 "enabled" : "disabled");
228 if (!(x2apic->Flags & ACPI_SRAT_CPU_ENABLED))
229 break;
230 if (x2apic->ApicId > MAX_APIC_ID) {
231 printf("SRAT: Ignoring local APIC ID %u (too high)\n",
232 x2apic->ApicId);
233 break;
234 }
235
236 KASSERT(!cpus[x2apic->ApicId].enabled,
237 ("Duplicate local APIC ID %u", x2apic->ApicId));
238 cpus[x2apic->ApicId].domain = x2apic->ProximityDomain;
239 cpus[x2apic->ApicId].enabled = 1;
240 break;
241 case ACPI_SRAT_TYPE_MEMORY_AFFINITY:
242 mem = (ACPI_SRAT_MEM_AFFINITY *)entry;
243 if (bootverbose)
244 printf(
245 "SRAT: Found memory domain %d addr 0x%jx len 0x%jx: %s\n",
246 mem->ProximityDomain, (uintmax_t)mem->BaseAddress,
247 (uintmax_t)mem->Length,
248 (mem->Flags & ACPI_SRAT_MEM_ENABLED) ?
249 "enabled" : "disabled");
250 if (!(mem->Flags & ACPI_SRAT_MEM_ENABLED))
251 break;
252 if (!overlaps_phys_avail(mem->BaseAddress,
253 mem->BaseAddress + mem->Length)) {
254 printf("SRAT: Ignoring memory at addr 0x%jx\n",
255 (uintmax_t)mem->BaseAddress);
256 break;
257 }
258 if (num_mem == VM_PHYSSEG_MAX) {
259 printf("SRAT: Too many memory regions\n");
260 *(int *)arg = ENXIO;
261 break;
262 }
263 slot = num_mem;
264 for (i = 0; i < num_mem; i++) {
265 if (mem_info[i].end <= mem->BaseAddress)
266 continue;
267 if (mem_info[i].start <
268 (mem->BaseAddress + mem->Length)) {
269 printf("SRAT: Overlapping memory entries\n");
270 *(int *)arg = ENXIO;
271 return;
272 }
273 slot = i;
274 }
275 for (i = num_mem; i > slot; i--)
276 mem_info[i] = mem_info[i - 1];
277 mem_info[slot].start = mem->BaseAddress;
278 mem_info[slot].end = mem->BaseAddress + mem->Length;
279 mem_info[slot].domain = mem->ProximityDomain;
280 num_mem++;
281 break;
282 }
283 }
284
285 /*
286 * Ensure each memory domain has at least one CPU and that each CPU
287 * has at least one memory domain.
288 */
289 static int
290 check_domains(void)
291 {
292 int found, i, j;
293
294 for (i = 0; i < num_mem; i++) {
295 found = 0;
296 for (j = 0; j <= MAX_APIC_ID; j++)
297 if (cpus[j].enabled &&
298 cpus[j].domain == mem_info[i].domain) {
299 cpus[j].has_memory = 1;
300 found++;
301 }
302 if (!found) {
303 printf("SRAT: No CPU found for memory domain %d\n",
304 mem_info[i].domain);
305 return (ENXIO);
306 }
307 }
308 for (i = 0; i <= MAX_APIC_ID; i++)
309 if (cpus[i].enabled && !cpus[i].has_memory) {
310 printf("SRAT: No memory found for CPU %d\n", i);
311 return (ENXIO);
312 }
313 return (0);
314 }
315
316 /*
317 * Check that the SRAT memory regions cover all of the regions in
318 * phys_avail[].
319 */
320 static int
321 check_phys_avail(void)
322 {
323 vm_paddr_t address;
324 int i, j;
325
326 /* j is the current offset into phys_avail[]. */
327 address = phys_avail[0];
328 j = 0;
329 for (i = 0; i < num_mem; i++) {
330 /*
331 * Consume as many phys_avail[] entries as fit in this
332 * region.
333 */
334 while (address >= mem_info[i].start &&
335 address <= mem_info[i].end) {
336 /*
337 * If we cover the rest of this phys_avail[] entry,
338 * advance to the next entry.
339 */
340 if (phys_avail[j + 1] <= mem_info[i].end) {
341 j += 2;
342 if (phys_avail[j] == 0 &&
343 phys_avail[j + 1] == 0) {
344 return (0);
345 }
346 address = phys_avail[j];
347 } else
348 address = mem_info[i].end + 1;
349 }
350 }
351 printf("SRAT: No memory region found for 0x%jx - 0x%jx\n",
352 (uintmax_t)phys_avail[j], (uintmax_t)phys_avail[j + 1]);
353 return (ENXIO);
354 }
355
356 /*
357 * Renumber the memory domains to be compact and zero-based if not
358 * already. Returns an error if there are too many domains.
359 */
360 static int
361 renumber_domains(void)
362 {
363 int i, j, slot;
364
365 /* Enumerate all the domains. */
366 ndomain = 0;
367 for (i = 0; i < num_mem; i++) {
368 /* See if this domain is already known. */
369 for (j = 0; j < ndomain; j++) {
370 if (domain_pxm[j] >= mem_info[i].domain)
371 break;
372 }
373 if (j < ndomain && domain_pxm[j] == mem_info[i].domain)
374 continue;
375
376 if (ndomain >= MAXMEMDOM) {
377 ndomain = 1;
378 printf("SRAT: Too many memory domains\n");
379 return (EFBIG);
380 }
381
382 /* Insert the new domain at slot 'j'. */
383 slot = j;
384 for (j = ndomain; j > slot; j--)
385 domain_pxm[j] = domain_pxm[j - 1];
386 domain_pxm[slot] = mem_info[i].domain;
387 ndomain++;
388 }
389
390 /* Renumber each domain to its index in the sorted 'domain_pxm' list. */
391 for (i = 0; i < ndomain; i++) {
392 /*
393 * If the domain is already the right value, no need
394 * to renumber.
395 */
396 if (domain_pxm[i] == i)
397 continue;
398
399 /* Walk the cpu[] and mem_info[] arrays to renumber. */
400 for (j = 0; j < num_mem; j++)
401 if (mem_info[j].domain == domain_pxm[i])
402 mem_info[j].domain = i;
403 for (j = 0; j <= MAX_APIC_ID; j++)
404 if (cpus[j].enabled && cpus[j].domain == domain_pxm[i])
405 cpus[j].domain = i;
406 }
407
408 return (0);
409 }
410
411 /*
412 * Look for an ACPI System Resource Affinity Table ("SRAT")
413 */
414 static int
415 parse_srat(void)
416 {
417 int error;
418
419 if (resource_disabled("srat", 0))
420 return (-1);
421
422 srat_physaddr = acpi_find_table(ACPI_SIG_SRAT);
423 if (srat_physaddr == 0)
424 return (-1);
425
426 /*
427 * Make a pass over the table to populate the cpus[] and
428 * mem_info[] tables.
429 */
430 srat = acpi_map_table(srat_physaddr, ACPI_SIG_SRAT);
431 error = 0;
432 srat_walk_table(srat_parse_entry, &error);
433 acpi_unmap_table(srat);
434 srat = NULL;
435 if (error || check_domains() != 0 || check_phys_avail() != 0 ||
436 renumber_domains() != 0) {
437 srat_physaddr = 0;
438 return (-1);
439 }
440
441 #ifdef VM_NUMA_ALLOC
442 /* Point vm_phys at our memory affinity table. */
443 vm_ndomains = ndomain;
444 mem_affinity = mem_info;
445 #endif
446
447 return (0);
448 }
449
450 static void
451 init_mem_locality(void)
452 {
453 int i;
454
455 /*
456 * For now, assume -1 == "no locality information for
457 * this pairing.
458 */
459 for (i = 0; i < MAXMEMDOM * MAXMEMDOM; i++)
460 vm_locality_table[i] = -1;
461 }
462
463 static void
464 parse_acpi_tables(void *dummy)
465 {
466
467 if (parse_srat() < 0)
468 return;
469 init_mem_locality();
470 (void) parse_slit();
471 }
472 SYSINIT(parse_acpi_tables, SI_SUB_VM - 1, SI_ORDER_FIRST, parse_acpi_tables,
473 NULL);
474
475 static void
476 srat_walk_table(acpi_subtable_handler *handler, void *arg)
477 {
478
479 acpi_walk_subtables(srat + 1, (char *)srat + srat->Header.Length,
480 handler, arg);
481 }
482
483 /*
484 * Setup per-CPU domain IDs.
485 */
486 static void
487 srat_set_cpus(void *dummy)
488 {
489 struct cpu_info *cpu;
490 struct pcpu *pc;
491 u_int i;
492
493 if (srat_physaddr == 0)
494 return;
495 for (i = 0; i < MAXCPU; i++) {
496 if (CPU_ABSENT(i))
497 continue;
498 pc = pcpu_find(i);
499 KASSERT(pc != NULL, ("no pcpu data for CPU %u", i));
500 cpu = &cpus[pc->pc_apic_id];
501 if (!cpu->enabled)
502 panic("SRAT: CPU with APIC ID %u is not known",
503 pc->pc_apic_id);
504 pc->pc_domain = cpu->domain;
505 CPU_SET(i, &cpuset_domain[cpu->domain]);
506 if (bootverbose)
507 printf("SRAT: CPU %u has memory domain %d\n", i,
508 cpu->domain);
509 }
510 }
511 SYSINIT(srat_set_cpus, SI_SUB_CPU, SI_ORDER_ANY, srat_set_cpus, NULL);
512
513 /*
514 * Map a _PXM value to a VM domain ID.
515 *
516 * Returns the domain ID, or -1 if no domain ID was found.
517 */
518 int
519 acpi_map_pxm_to_vm_domainid(int pxm)
520 {
521 int i;
522
523 for (i = 0; i < ndomain; i++) {
524 if (domain_pxm[i] == pxm)
525 return (i);
526 }
527
528 return (-1);
529 }
530
531 #else /* MAXMEMDOM == 1 */
532
533 int
534 acpi_map_pxm_to_vm_domainid(int pxm)
535 {
536
537 return (-1);
538 }
539
540 #endif /* MAXMEMDOM > 1 */

Properties

Name Value
svn:keywords MidnightBSD=%H