1 /* $NetBSD: gicv3_its.c,v 1.41 2025/01/28 21:20:45 jmcneill Exp $ */
2
3 /*-
4 * Copyright (c) 2018 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jared McNeill <jmcneill@invisible.ca>.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 #define _INTR_PRIVATE
33
34 #include <sys/cdefs.h>
35 __KERNEL_RCSID(0, "$NetBSD: gicv3_its.c,v 1.41 2025/01/28 21:20:45 jmcneill Exp $");
36
37 #include <sys/param.h>
38 #include <sys/kmem.h>
39 #include <sys/bus.h>
40 #include <sys/cpu.h>
41 #include <sys/bitops.h>
42
43 #include <uvm/uvm.h>
44
45 #include <dev/pci/pcireg.h>
46 #include <dev/pci/pcivar.h>
47
48 #include <machine/cpufunc.h>
49
50 #include <arm/pic/picvar.h>
51 #include <arm/cortex/gicv3_its.h>
52
53 #ifdef ITS_DEBUG
54 #define DPRINTF(x) printf x
55 #else
56 #define DPRINTF(x) __nothing
57 #endif
58
59 /*
60 * ITS translation table sizes
61 */
62 #define GITS_COMMANDS_SIZE 0x1000
63 #define GITS_COMMANDS_ALIGN 0x10000
64
65 #define GITS_ITT_ALIGN 0x100
66
67 #define GITS_INDIRECT_ENTRY_SIZE 8
68
69 /*
70 * IIDR values used for errata
71 */
72 #define GITS_IIDR_PID_CAVIUM_THUNDERX 0xa1
73 #define GITS_IIDR_IMP_CAVIUM 0x34c
74 #define GITS_IIDR_CAVIUM_ERRATA_MASK (GITS_IIDR_Implementor|GITS_IIDR_ProductID|GITS_IIDR_Variant)
75 #define GITS_IIDR_CAVIUM_ERRATA_VALUE \
76 (__SHIFTIN(GITS_IIDR_IMP_CAVIUM, GITS_IIDR_Implementor) | \
77 __SHIFTIN(GITS_IIDR_PID_CAVIUM_THUNDERX, GITS_IIDR_ProductID) | \
78 __SHIFTIN(0, GITS_IIDR_Variant))
79
80 static const char * gits_cache_type[] = {
81 [GITS_Cache_DEVICE_nGnRnE] = "Device-nGnRnE",
82 [GITS_Cache_NORMAL_NC] = "Non-cacheable",
83 [GITS_Cache_NORMAL_RA_WT] = "Cacheable RA WT",
84 [GITS_Cache_NORMAL_RA_WB] = "Cacheable RA WB",
85 [GITS_Cache_NORMAL_WA_WT] = "Cacheable WA WT",
86 [GITS_Cache_NORMAL_WA_WB] = "Cacheable WA WB",
87 [GITS_Cache_NORMAL_RA_WA_WT] = "Cacheable RA WA WT",
88 [GITS_Cache_NORMAL_RA_WA_WB] = "Cacheable RA WA WB",
89 };
90
91 static const char * gits_share_type[] = {
92 [GITS_Shareability_NS] = "Non-shareable",
93 [GITS_Shareability_IS] = "Inner shareable",
94 [GITS_Shareability_OS] = "Outer shareable",
95 [3] = "(Reserved)",
96 };
97
98 static inline uint32_t
gits_read_4(struct gicv3_its * its,bus_size_t reg)99 gits_read_4(struct gicv3_its *its, bus_size_t reg)
100 {
101 return bus_space_read_4(its->its_bst, its->its_bsh, reg);
102 }
103
104 static inline void
gits_write_4(struct gicv3_its * its,bus_size_t reg,uint32_t val)105 gits_write_4(struct gicv3_its *its, bus_size_t reg, uint32_t val)
106 {
107 bus_space_write_4(its->its_bst, its->its_bsh, reg, val);
108 }
109
110 static inline uint64_t
gits_read_8(struct gicv3_its * its,bus_size_t reg)111 gits_read_8(struct gicv3_its *its, bus_size_t reg)
112 {
113 return bus_space_read_8(its->its_bst, its->its_bsh, reg);
114 }
115
116 static inline void
gits_write_8(struct gicv3_its * its,bus_size_t reg,uint64_t val)117 gits_write_8(struct gicv3_its *its, bus_size_t reg, uint64_t val)
118 {
119 bus_space_write_8(its->its_bst, its->its_bsh, reg, val);
120 }
121
122 static int
gits_command(struct gicv3_its * its,const struct gicv3_its_command * cmd)123 gits_command(struct gicv3_its *its, const struct gicv3_its_command *cmd)
124 {
125 uint64_t cwriter, creadr;
126 u_int woff;
127
128 creadr = gits_read_8(its, GITS_CREADR);
129 if (ISSET(creadr, GITS_CREADR_Stalled)) {
130 DPRINTF(("ITS: stalled! GITS_CREADR = 0x%lx\n", creadr));
131 return EIO;
132 }
133
134 cwriter = gits_read_8(its, GITS_CWRITER);
135 woff = cwriter & GITS_CWRITER_Offset;
136
137 uint64_t *dw = (uint64_t *)(its->its_cmd.base + woff);
138 for (int i = 0; i < __arraycount(cmd->dw); i++) {
139 dw[i] = htole64(cmd->dw[i]);
140 DPRINTF(("ITS: dw[%u] = 0x%016lx\n", i, cmd->dw[i]));
141 }
142
143 if (its->its_cmd_flush) {
144 cpu_dcache_wb_range((vaddr_t)dw, sizeof(cmd->dw));
145 }
146 dsb(sy);
147
148 woff += sizeof(cmd->dw);
149 if (woff == its->its_cmd.len)
150 woff = 0;
151
152 gits_write_8(its, GITS_CWRITER, woff);
153
154 return 0;
155 }
156
157 static int
gits_command_mapc(struct gicv3_its * its,uint16_t icid,uint64_t rdbase,bool v)158 gits_command_mapc(struct gicv3_its *its, uint16_t icid, uint64_t rdbase, bool v)
159 {
160 struct gicv3_its_command cmd;
161
162 KASSERT((rdbase & 0xffff) == 0);
163
164 /*
165 * Map a collection table entry (ICID) to the target redistributor (RDbase).
166 */
167 memset(&cmd, 0, sizeof(cmd));
168 cmd.dw[0] = GITS_CMD_MAPC;
169 cmd.dw[2] = icid;
170 if (v) {
171 cmd.dw[2] |= rdbase;
172 cmd.dw[2] |= __BIT(63);
173 }
174
175 DPRINTF(("ITS #%u: MAPC icid 0x%x rdbase 0x%lx valid %u\n",
176 its->its_id, icid, rdbase, v));
177
178 return gits_command(its, &cmd);
179 }
180
181 static int
gits_command_mapd(struct gicv3_its * its,uint32_t deviceid,uint64_t itt_addr,u_int size,bool v)182 gits_command_mapd(struct gicv3_its *its, uint32_t deviceid, uint64_t itt_addr, u_int size, bool v)
183 {
184 struct gicv3_its_command cmd;
185
186 KASSERT((itt_addr & 0xff) == 0);
187
188 /*
189 * Map a device table entry (DeviceID) to its associated ITT (ITT_addr).
190 */
191 memset(&cmd, 0, sizeof(cmd));
192 cmd.dw[0] = GITS_CMD_MAPD | ((uint64_t)deviceid << 32);
193 if (v) {
194 cmd.dw[1] = uimax(1, size) - 1;
195 cmd.dw[2] = itt_addr | __BIT(63);
196 }
197
198 DPRINTF(("ITS #%u: MAPD deviceid 0x%x itt_addr 0x%lx size %u valid %u\n",
199 its->its_id, deviceid, itt_addr, size, v));
200
201 return gits_command(its, &cmd);
202 }
203
204 static int
gits_command_mapti(struct gicv3_its * its,uint32_t deviceid,uint32_t eventid,uint32_t pintid,uint16_t icid)205 gits_command_mapti(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid, uint32_t pintid, uint16_t icid)
206 {
207 struct gicv3_its_command cmd;
208
209 /*
210 * Map the event defined by EventID and DeviceID to its associated ITE, defined by ICID and pINTID
211 * in the ITT associated with DeviceID.
212 */
213 memset(&cmd, 0, sizeof(cmd));
214 cmd.dw[0] = GITS_CMD_MAPTI | ((uint64_t)deviceid << 32);
215 cmd.dw[1] = eventid | ((uint64_t)pintid << 32);
216 cmd.dw[2] = icid;
217
218 DPRINTF(("ITS #%u: MAPTI deviceid 0x%x eventid 0x%x pintid 0x%x icid 0x%x\n",
219 its->its_id, deviceid, eventid, pintid, icid));
220
221 return gits_command(its, &cmd);
222 }
223
224 static int
gits_command_movi(struct gicv3_its * its,uint32_t deviceid,uint32_t eventid,uint16_t icid)225 gits_command_movi(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid, uint16_t icid)
226 {
227 struct gicv3_its_command cmd;
228
229 /*
230 * Update the ICID field in the ITT entry for the event defined by DeviceID and
231 * EventID.
232 */
233 memset(&cmd, 0, sizeof(cmd));
234 cmd.dw[0] = GITS_CMD_MOVI | ((uint64_t)deviceid << 32);
235 cmd.dw[1] = eventid;
236 cmd.dw[2] = icid;
237
238 DPRINTF(("ITS #%u: MOVI deviceid 0x%x eventid 0x%x icid 0x%x\n",
239 its->its_id, deviceid, eventid, icid));
240
241 return gits_command(its, &cmd);
242 }
243
244 static int
gits_command_inv(struct gicv3_its * its,uint32_t deviceid,uint32_t eventid)245 gits_command_inv(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid)
246 {
247 struct gicv3_its_command cmd;
248
249 /*
250 * Ensure any caching in the redistributors associated with the specified
251 * EventID is consistent with the LPI configuration tables.
252 */
253 memset(&cmd, 0, sizeof(cmd));
254 cmd.dw[0] = GITS_CMD_INV | ((uint64_t)deviceid << 32);
255 cmd.dw[1] = eventid;
256
257 DPRINTF(("ITS #%u: INV deviceid 0x%x eventid 0x%x\n",
258 its->its_id, deviceid, eventid));
259
260 return gits_command(its, &cmd);
261 }
262
263 static int
gits_command_invall(struct gicv3_its * its,uint16_t icid)264 gits_command_invall(struct gicv3_its *its, uint16_t icid)
265 {
266 struct gicv3_its_command cmd;
267
268 /*
269 * Ensure any caching associated with this ICID is consistent with LPI
270 * configuration tables for all redistributors.
271 */
272 memset(&cmd, 0, sizeof(cmd));
273 cmd.dw[0] = GITS_CMD_INVALL;
274 cmd.dw[2] = icid;
275
276 DPRINTF(("ITS #%u: INVALL icid 0x%x\n", its->its_id, icid));
277
278 return gits_command(its, &cmd);
279 }
280
281 static int
gits_command_sync(struct gicv3_its * its,uint64_t rdbase)282 gits_command_sync(struct gicv3_its *its, uint64_t rdbase)
283 {
284 struct gicv3_its_command cmd;
285
286 KASSERT((rdbase & 0xffff) == 0);
287
288 /*
289 * Ensure all outstanding ITS operations associated with physical interrupts
290 * for the specified redistributor (RDbase) are globally observed before
291 * further ITS commands are executed.
292 */
293 memset(&cmd, 0, sizeof(cmd));
294 cmd.dw[0] = GITS_CMD_SYNC;
295 cmd.dw[2] = rdbase;
296
297 DPRINTF(("ITS #%u: SYNC rdbase 0x%lx\n", its->its_id, rdbase));
298
299 return gits_command(its, &cmd);
300 }
301
302 #if 0
303 static int
304 gits_command_int(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid)
305 {
306 struct gicv3_its_command cmd;
307
308 /*
309 * Translate the deviceid and eventid into an icid and pintid through
310 * the device table and ITT. Mark the pintid as pending
311 * on the redistributor.
312 * If the interrupt is not configured the command queue stalls.
313 */
314 memset(&cmd, 0, sizeof(cmd));
315 cmd.dw[0] = GITS_CMD_INT | ((uint64_t)deviceid << 32);
316 cmd.dw[1] = eventid;
317
318 DPRINTF(("ITS #%u: INT deviceid 0x%x eventid 0x%x\n",
319 its->its_id, deviceid, eventid));
320
321 return gits_command(its, &cmd);
322 }
323 #endif
324
325 static int
gits_wait(struct gicv3_its * its)326 gits_wait(struct gicv3_its *its)
327 {
328 u_int woff, roff;
329 int retry = 100000;
330
331 /*
332 * The ITS command queue is empty when CWRITER and CREADR specify the
333 * same base address offset value.
334 */
335 for (retry = 1000; retry > 0; retry--) {
336 woff = gits_read_8(its, GITS_CWRITER) & GITS_CWRITER_Offset;
337 roff = gits_read_8(its, GITS_CREADR) & GITS_CREADR_Offset;
338 if (woff == roff)
339 break;
340 delay(100);
341 }
342 if (retry == 0) {
343 device_printf(its->its_gic->sc_dev,
344 "ITS command queue timeout! CREADR=0x%lx CWRITER=0x%lx\n",
345 gits_read_8(its, GITS_CREADR), gits_read_8(its, GITS_CWRITER));
346 return ETIMEDOUT;
347 }
348
349 return 0;
350 }
351
352 static int
gicv3_its_msi_alloc_lpi(struct gicv3_its * its,const struct pci_attach_args * pa)353 gicv3_its_msi_alloc_lpi(struct gicv3_its *its,
354 const struct pci_attach_args *pa)
355 {
356 struct pci_attach_args *new_pa;
357 vmem_addr_t n;
358
359 KASSERT(its->its_gic->sc_lpi_pool != NULL);
360
361 if (vmem_alloc(its->its_gic->sc_lpi_pool, 1, VM_INSTANTFIT|VM_SLEEP, &n) != 0)
362 return -1;
363
364 KASSERT(its->its_pa[n] == NULL);
365
366 new_pa = kmem_alloc(sizeof(*new_pa), KM_SLEEP);
367 memcpy(new_pa, pa, sizeof(*new_pa));
368 its->its_pa[n] = new_pa;
369 return n + its->its_pic->pic_irqbase;
370 }
371
372 static void
gicv3_its_msi_free_lpi(struct gicv3_its * its,int lpi)373 gicv3_its_msi_free_lpi(struct gicv3_its *its, int lpi)
374 {
375 struct pci_attach_args *pa;
376
377 KASSERT(its->its_gic->sc_lpi_pool != NULL);
378 KASSERT(lpi >= its->its_pic->pic_irqbase);
379
380 pa = its->its_pa[lpi - its->its_pic->pic_irqbase];
381 its->its_pa[lpi - its->its_pic->pic_irqbase] = NULL;
382 kmem_free(pa, sizeof(*pa));
383
384 vmem_free(its->its_gic->sc_lpi_pool, lpi - its->its_pic->pic_irqbase, 1);
385 }
386
387 static uint32_t
gicv3_its_devid(pci_chipset_tag_t pc,pcitag_t tag)388 gicv3_its_devid(pci_chipset_tag_t pc, pcitag_t tag)
389 {
390 uint32_t devid;
391 int b, d, f;
392
393 pci_decompose_tag(pc, tag, &b, &d, &f);
394
395 devid = (b << 8) | (d << 3) | f;
396
397 return pci_get_devid(pc, devid);
398 }
399
400 static int
gicv3_its_device_map(struct gicv3_its * its,uint32_t devid,u_int count)401 gicv3_its_device_map(struct gicv3_its *its, uint32_t devid, u_int count)
402 {
403 struct gicv3_its_device *dev;
404 struct gicv3_its_table *itstab = &its->its_tab_device;
405 u_int vectors;
406 int error;
407
408 vectors = MAX(2, count);
409 while (!powerof2(vectors))
410 vectors++;
411
412 const uint64_t typer = gits_read_8(its, GITS_TYPER);
413 const u_int itt_entry_size = __SHIFTOUT(typer, GITS_TYPER_ITT_entry_size) + 1;
414 const u_int itt_size = roundup(uimax(vectors, 2) * itt_entry_size, GITS_ITT_ALIGN);
415
416 LIST_FOREACH(dev, &its->its_devices, dev_list)
417 if (dev->dev_id == devid) {
418 return itt_size <= dev->dev_size ? 0 : EEXIST;
419 }
420
421 if (itstab->tab_indirect) {
422 uint64_t *l1_tab = itstab->tab_l1;
423 const u_int index = devid / itstab->tab_l2_num_ids;
424
425 if ((l1_tab[index] & GITS_BASER_Valid) == 0) {
426 /* Need to allocate the L2 table. */
427 struct gicv3_its_page_table *pt;
428
429 pt = kmem_alloc(sizeof(*pt), KM_SLEEP);
430 pt->pt_index = index;
431 gicv3_dma_alloc(its->its_gic, &pt->pt_dma, itstab->tab_l2_entry_size,
432 itstab->tab_page_size);
433 LIST_INSERT_HEAD(&itstab->tab_pt, pt, pt_list);
434
435 if (!itstab->tab_shareable) {
436 cpu_dcache_wb_range((vaddr_t)pt->pt_dma.base,
437 itstab->tab_l2_entry_size);
438 }
439 l1_tab[index] = pt->pt_dma.segs[0].ds_addr | GITS_BASER_Valid;
440 if (!itstab->tab_shareable) {
441 cpu_dcache_wb_range((vaddr_t)&l1_tab[index],
442 sizeof(l1_tab[index]));
443 }
444 dsb(sy);
445
446 DPRINTF(("ITS: Allocated L2 entry at index %u\n", index));
447 }
448 }
449
450 dev = kmem_alloc(sizeof(*dev), KM_SLEEP);
451 dev->dev_id = devid;
452 dev->dev_size = itt_size;
453 gicv3_dma_alloc(its->its_gic, &dev->dev_itt, itt_size, GITS_ITT_ALIGN);
454 LIST_INSERT_HEAD(&its->its_devices, dev, dev_list);
455
456 if (its->its_cmd_flush) {
457 cpu_dcache_wb_range((vaddr_t)dev->dev_itt.base, itt_size);
458 }
459 dsb(sy);
460
461 /*
462 * Map the device to the ITT
463 */
464 const u_int size = __SHIFTOUT(typer, GITS_TYPER_ID_bits) + 1;
465 mutex_enter(its->its_lock);
466 error = gits_command_mapd(its, devid, dev->dev_itt.segs[0].ds_addr, size, true);
467 if (error == 0) {
468 error = gits_wait(its);
469 }
470 mutex_exit(its->its_lock);
471
472 return error;
473 }
474
475 static void
gicv3_its_msi_enable(struct gicv3_its * its,int lpi,int count)476 gicv3_its_msi_enable(struct gicv3_its *its, int lpi, int count)
477 {
478 const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase];
479 pci_chipset_tag_t pc = pa->pa_pc;
480 pcitag_t tag = pa->pa_tag;
481 pcireg_t ctl;
482 int off;
483
484 if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL))
485 panic("gicv3_its_msi_enable: device is not MSI-capable");
486
487 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
488 ctl &= ~PCI_MSI_CTL_MME_MASK;
489 ctl |= __SHIFTIN(ilog2(count), PCI_MSI_CTL_MME_MASK);
490 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);
491
492 const uint64_t addr = its->its_base + GITS_TRANSLATER;
493 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
494 if (ctl & PCI_MSI_CTL_64BIT_ADDR) {
495 pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_LO,
496 addr & 0xffffffff);
497 pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_HI,
498 (addr >> 32) & 0xffffffff);
499 pci_conf_write(pc, tag, off + PCI_MSI_MDATA64,
500 lpi - its->its_pic->pic_irqbase);
501 } else {
502 KASSERT((addr >> 32) == 0);
503 pci_conf_write(pc, tag, off + PCI_MSI_MADDR,
504 addr & 0xffffffff);
505 pci_conf_write(pc, tag, off + PCI_MSI_MDATA,
506 lpi - its->its_pic->pic_irqbase);
507 }
508 ctl |= PCI_MSI_CTL_MSI_ENABLE;
509 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);
510 }
511
512 static void
gicv3_its_msi_disable(struct gicv3_its * its,int lpi)513 gicv3_its_msi_disable(struct gicv3_its *its, int lpi)
514 {
515 const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase];
516 pci_chipset_tag_t pc = pa->pa_pc;
517 pcitag_t tag = pa->pa_tag;
518 pcireg_t ctl;
519 int off;
520
521 if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL))
522 panic("gicv3_its_msi_enable: device is not MSI-capable");
523
524 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
525 ctl &= ~PCI_MSI_CTL_MSI_ENABLE;
526 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);
527 }
528
529 static void
gicv3_its_msix_enable(struct gicv3_its * its,int lpi,int msix_vec,bus_space_tag_t bst,bus_space_handle_t bsh)530 gicv3_its_msix_enable(struct gicv3_its *its, int lpi, int msix_vec,
531 bus_space_tag_t bst, bus_space_handle_t bsh)
532 {
533 const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase];
534 pci_chipset_tag_t pc = pa->pa_pc;
535 pcitag_t tag = pa->pa_tag;
536 pcireg_t ctl;
537 uint32_t val;
538 int off;
539
540 if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL))
541 panic("gicv3_its_msix_enable: device is not MSI-X-capable");
542
543 const uint64_t addr = its->its_base + GITS_TRANSLATER;
544 const uint64_t entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec;
545 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_LO, (uint32_t)addr);
546 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_HI, (uint32_t)(addr >> 32));
547 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_DATA, lpi - its->its_pic->pic_irqbase);
548 val = bus_space_read_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL);
549 val &= ~PCI_MSIX_VECTCTL_MASK;
550 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, val);
551
552 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL);
553 ctl |= PCI_MSIX_CTL_ENABLE;
554 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);
555 }
556
557 static void
gicv3_its_msix_disable(struct gicv3_its * its,int lpi)558 gicv3_its_msix_disable(struct gicv3_its *its, int lpi)
559 {
560 const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase];
561 pci_chipset_tag_t pc = pa->pa_pc;
562 pcitag_t tag = pa->pa_tag;
563 pcireg_t ctl;
564 int off;
565
566 if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL))
567 panic("gicv3_its_msix_disable: device is not MSI-X-capable");
568
569 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL);
570 ctl &= ~PCI_MSIX_CTL_ENABLE;
571 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);
572 }
573
574 static pci_intr_handle_t *
gicv3_its_msi_alloc(struct arm_pci_msi * msi,int * count,const struct pci_attach_args * pa,bool exact)575 gicv3_its_msi_alloc(struct arm_pci_msi *msi, int *count,
576 const struct pci_attach_args *pa, bool exact)
577 {
578 struct gicv3_its * const its = msi->msi_priv;
579 struct cpu_info * const ci = cpu_lookup(0);
580 pci_intr_handle_t *vectors;
581 int n, off, error;
582
583 if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSI, &off, NULL))
584 return NULL;
585
586 const uint64_t typer = gits_read_8(its, GITS_TYPER);
587 const u_int id_bits = __SHIFTOUT(typer, GITS_TYPER_ID_bits) + 1;
588 if (*count == 0 || *count > (1 << id_bits))
589 return NULL;
590
591 const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag);
592
593 if (gicv3_its_device_map(its, devid, *count) != 0)
594 return NULL;
595
596 vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP);
597 mutex_enter(its->its_lock);
598 for (n = 0; n < *count; n++) {
599 const int lpi = gicv3_its_msi_alloc_lpi(its, pa);
600 KASSERT(lpi >= 0);
601 vectors[n] = ARM_PCI_INTR_MSI |
602 __SHIFTIN(lpi, ARM_PCI_INTR_IRQ) |
603 __SHIFTIN(n, ARM_PCI_INTR_MSI_VEC) |
604 __SHIFTIN(msi->msi_id, ARM_PCI_INTR_FRAME);
605
606 if (n == 0)
607 gicv3_its_msi_enable(its, lpi, *count);
608
609 /*
610 * Record devid and target PE
611 */
612 its->its_devid[lpi - its->its_pic->pic_irqbase] = devid;
613 its->its_targets[lpi - its->its_pic->pic_irqbase] = ci;
614
615 /*
616 * Map event
617 */
618 gits_command_mapti(its, devid, lpi - its->its_pic->pic_irqbase, lpi, cpu_index(ci));
619 gits_command_sync(its, its->its_rdbase[cpu_index(ci)]);
620 }
621 error = gits_wait(its);
622 mutex_exit(its->its_lock);
623
624 if (error != 0) {
625 kmem_free(vectors, sizeof(*vectors) * *count);
626 vectors = NULL;
627 }
628
629 return vectors;
630 }
631
632 static pci_intr_handle_t *
gicv3_its_msix_alloc(struct arm_pci_msi * msi,u_int * table_indexes,int * count,const struct pci_attach_args * pa,bool exact)633 gicv3_its_msix_alloc(struct arm_pci_msi *msi, u_int *table_indexes, int *count,
634 const struct pci_attach_args *pa, bool exact)
635 {
636 struct gicv3_its * const its = msi->msi_priv;
637 struct cpu_info *ci = cpu_lookup(0);
638 pci_intr_handle_t *vectors;
639 bus_space_tag_t bst;
640 bus_space_handle_t bsh;
641 bus_size_t bsz;
642 uint32_t table_offset, table_size;
643 int n, off, bar, error;
644 pcireg_t tbl;
645
646 if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, &off, NULL))
647 return NULL;
648
649 const uint64_t typer = gits_read_8(its, GITS_TYPER);
650 const u_int id_bits = __SHIFTOUT(typer, GITS_TYPER_ID_bits) + 1;
651 if (*count == 0 || *count > (1 << id_bits))
652 return NULL;
653
654 tbl = pci_conf_read(pa->pa_pc, pa->pa_tag, off + PCI_MSIX_TBLOFFSET);
655 bar = PCI_BAR0 + (4 * (tbl & PCI_MSIX_TBLBIR_MASK));
656 table_offset = tbl & PCI_MSIX_TBLOFFSET_MASK;
657 table_size = pci_msix_count(pa->pa_pc, pa->pa_tag) * PCI_MSIX_TABLE_ENTRY_SIZE;
658 if (table_size == 0)
659 return NULL;
660
661 error = pci_mapreg_submap(pa, bar, pci_mapreg_type(pa->pa_pc, pa->pa_tag, bar),
662 BUS_SPACE_MAP_LINEAR, roundup(table_size, PAGE_SIZE), table_offset,
663 &bst, &bsh, NULL, &bsz);
664 if (error)
665 return NULL;
666
667 const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag);
668
669 if (gicv3_its_device_map(its, devid, *count) != 0) {
670 bus_space_unmap(bst, bsh, bsz);
671 return NULL;
672 }
673
674 vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP);
675 mutex_enter(its->its_lock);
676 for (n = 0; n < *count; n++) {
677 const int lpi = gicv3_its_msi_alloc_lpi(its, pa);
678 KASSERT(lpi >= 0);
679 const int msix_vec = table_indexes ? table_indexes[n] : n;
680 vectors[msix_vec] = ARM_PCI_INTR_MSIX |
681 __SHIFTIN(lpi, ARM_PCI_INTR_IRQ) |
682 __SHIFTIN(msix_vec, ARM_PCI_INTR_MSI_VEC) |
683 __SHIFTIN(msi->msi_id, ARM_PCI_INTR_FRAME);
684
685 gicv3_its_msix_enable(its, lpi, msix_vec, bst, bsh);
686
687 /*
688 * Record devid and target PE
689 */
690 its->its_devid[lpi - its->its_pic->pic_irqbase] = devid;
691 its->its_targets[lpi - its->its_pic->pic_irqbase] = ci;
692
693 /*
694 * Map event
695 */
696 gits_command_mapti(its, devid, lpi - its->its_pic->pic_irqbase, lpi, cpu_index(ci));
697 gits_command_sync(its, its->its_rdbase[cpu_index(ci)]);
698 }
699 gits_wait(its);
700 mutex_exit(its->its_lock);
701
702 bus_space_unmap(bst, bsh, bsz);
703
704 return vectors;
705 }
706
707 static void *
gicv3_its_msi_intr_establish(struct arm_pci_msi * msi,pci_intr_handle_t ih,int ipl,int (* func)(void *),void * arg,const char * xname)708 gicv3_its_msi_intr_establish(struct arm_pci_msi *msi,
709 pci_intr_handle_t ih, int ipl, int (*func)(void *), void *arg, const char *xname)
710 {
711 struct gicv3_its * const its = msi->msi_priv;
712 void *intrh;
713
714 const int lpi = __SHIFTOUT(ih, ARM_PCI_INTR_IRQ);
715 const int mpsafe = (ih & ARM_PCI_INTR_MPSAFE) ? IST_MPSAFE : 0;
716
717 intrh = pic_establish_intr(its->its_pic, lpi - its->its_pic->pic_irqbase, ipl,
718 IST_EDGE | mpsafe, func, arg, xname);
719 if (intrh == NULL)
720 return NULL;
721
722 /* Invalidate LPI configuration tables */
723 KASSERT(its->its_pa[lpi - its->its_pic->pic_irqbase] != NULL);
724 const uint32_t devid = its->its_devid[lpi - its->its_pic->pic_irqbase];
725 gits_command_inv(its, devid, lpi - its->its_pic->pic_irqbase);
726
727 return intrh;
728 }
729
730 static void
gicv3_its_msi_intr_release(struct arm_pci_msi * msi,pci_intr_handle_t * pih,int count)731 gicv3_its_msi_intr_release(struct arm_pci_msi *msi, pci_intr_handle_t *pih,
732 int count)
733 {
734 struct gicv3_its * const its = msi->msi_priv;
735 int n;
736
737 for (n = 0; n < count; n++) {
738 const int lpi = __SHIFTOUT(pih[n], ARM_PCI_INTR_IRQ);
739 KASSERT(lpi >= its->its_pic->pic_irqbase);
740 if (pih[n] & ARM_PCI_INTR_MSIX)
741 gicv3_its_msix_disable(its, lpi);
742 if (pih[n] & ARM_PCI_INTR_MSI)
743 gicv3_its_msi_disable(its, lpi);
744 gicv3_its_msi_free_lpi(its, lpi);
745 its->its_targets[lpi - its->its_pic->pic_irqbase] = NULL;
746 its->its_devid[lpi - its->its_pic->pic_irqbase] = 0;
747 struct intrsource * const is =
748 its->its_pic->pic_sources[lpi - its->its_pic->pic_irqbase];
749 if (is != NULL)
750 pic_disestablish_source(is);
751 }
752 }
753
754 static void
gicv3_its_command_init(struct gicv3_softc * sc,struct gicv3_its * its)755 gicv3_its_command_init(struct gicv3_softc *sc, struct gicv3_its *its)
756 {
757 uint64_t cbaser, tmp;
758
759 gicv3_dma_alloc(sc, &its->its_cmd, GITS_COMMANDS_SIZE, GITS_COMMANDS_ALIGN);
760 if (its->its_cmd_flush) {
761 cpu_dcache_wb_range((vaddr_t)its->its_cmd.base, GITS_COMMANDS_SIZE);
762 }
763 dsb(sy);
764
765 KASSERT((gits_read_4(its, GITS_CTLR) & GITS_CTLR_Enabled) == 0);
766 KASSERT((gits_read_4(its, GITS_CTLR) & GITS_CTLR_Quiescent) != 0);
767
768 cbaser = its->its_cmd.segs[0].ds_addr;
769 cbaser |= __SHIFTIN((its->its_cmd.len / 4096) - 1, GITS_CBASER_Size);
770 cbaser |= GITS_CBASER_Valid;
771
772 cbaser |= __SHIFTIN(GITS_Cache_NORMAL_WA_WB, GITS_CBASER_InnerCache);
773 cbaser |= __SHIFTIN(GITS_Shareability_IS, GITS_CBASER_Shareability);
774 gits_write_8(its, GITS_CBASER, cbaser);
775
776 tmp = gits_read_8(its, GITS_CBASER);
777 if (__SHIFTOUT(tmp, GITS_CBASER_Shareability) != GITS_Shareability_IS) {
778 if (__SHIFTOUT(tmp, GITS_CBASER_InnerCache) == GITS_Shareability_NS) {
779 cbaser &= ~GITS_CBASER_InnerCache;
780 cbaser |= __SHIFTIN(GITS_Cache_NORMAL_NC, GITS_CBASER_InnerCache);
781 cbaser &= ~GITS_CBASER_Shareability;
782 cbaser |= __SHIFTIN(GITS_Shareability_NS, GITS_CBASER_Shareability);
783 gits_write_8(its, GITS_CBASER, cbaser);
784 }
785
786 its->its_cmd_flush = true;
787 }
788 aprint_normal_dev(sc->sc_dev, "ITS command table @ %#lx/%#lx, %s, %s\n",
789 its->its_cmd.segs[0].ds_addr, its->its_cmd.len,
790 gits_cache_type[__SHIFTOUT(cbaser, GITS_BASER_InnerCache)],
791 gits_share_type[__SHIFTOUT(cbaser, GITS_BASER_Shareability)]);
792
793 gits_write_8(its, GITS_CWRITER, 0);
794 }
795
796 static void
gicv3_its_table_params(struct gicv3_softc * sc,struct gicv3_its * its,u_int * devbits,u_int * innercache,u_int * share)797 gicv3_its_table_params(struct gicv3_softc *sc, struct gicv3_its *its,
798 u_int *devbits, u_int *innercache, u_int *share)
799 {
800
801 const uint64_t typer = gits_read_8(its, GITS_TYPER);
802 const uint32_t iidr = gits_read_4(its, GITS_IIDR);
803
804 /* Default values */
805 *devbits = __SHIFTOUT(typer, GITS_TYPER_Devbits) + 1;
806 *innercache = GITS_Cache_NORMAL_WA_WB;
807 *share = GITS_Shareability_IS;
808
809 /* Cavium ThunderX errata */
810 if ((iidr & GITS_IIDR_CAVIUM_ERRATA_MASK) == GITS_IIDR_CAVIUM_ERRATA_VALUE) {
811 *devbits = 20; /* 8Mb */
812 *innercache = GITS_Cache_DEVICE_nGnRnE;
813 aprint_normal_dev(sc->sc_dev, "Cavium ThunderX errata detected\n");
814 }
815 }
816
817 static bool
gicv3_its_table_probe_indirect(struct gicv3_its * its,int tab)818 gicv3_its_table_probe_indirect(struct gicv3_its *its, int tab)
819 {
820 uint64_t baser;
821
822 baser = gits_read_8(its, GITS_BASERn(tab));
823 baser |= GITS_BASER_Indirect;
824 gits_write_8(its, GITS_BASERn(tab), baser);
825
826 baser = gits_read_8(its, GITS_BASERn(tab));
827
828 return (baser & GITS_BASER_Indirect) != 0;
829 }
830
831 static void
gicv3_its_table_init(struct gicv3_softc * sc,struct gicv3_its * its)832 gicv3_its_table_init(struct gicv3_softc *sc, struct gicv3_its *its)
833 {
834 u_int page_size, table_align;
835 u_int devbits, innercache, share;
836 const char *table_type;
837 uint64_t baser;
838 int tab;
839
840 gicv3_its_table_params(sc, its, &devbits, &innercache, &share);
841
842 DPRINTF(("ITS: devbits = %u\n", devbits));
843
844 for (tab = 0; tab < 8; tab++) {
845 struct gicv3_its_table *itstab;
846 bool indirect = false;
847 uint64_t l1_entry_size, l2_entry_size;
848 uint64_t l1_num_ids, l2_num_ids;
849 uint64_t table_size;
850
851 baser = gits_read_8(its, GITS_BASERn(tab));
852
853 l1_entry_size = __SHIFTOUT(baser, GITS_BASER_Entry_Size) + 1;
854 l2_entry_size = 0;
855 l2_num_ids = 0;
856
857 switch (__SHIFTOUT(baser, GITS_BASER_Page_Size)) {
858 case GITS_Page_Size_64KB:
859 page_size = 65536;
860 break;
861 case GITS_Page_Size_16KB:
862 page_size = 16384;
863 break;
864 case GITS_Page_Size_4KB:
865 default:
866 page_size = 4096;
867 }
868 table_align = page_size;
869
870 switch (__SHIFTOUT(baser, GITS_BASER_Type)) {
871 case GITS_Type_Devices:
872 /*
873 * Table size scales with the width of the DeviceID.
874 */
875 l1_num_ids = 1ULL << devbits;
876 DPRINTF(("ITS: l1_num_ids = %lu\n", l1_num_ids));
877 indirect =
878 gicv3_its_table_probe_indirect(its, tab);
879 if (indirect) {
880 DPRINTF(("ITS: indirect\n"));
881 l2_entry_size = l1_entry_size;
882 l2_num_ids = page_size / l2_entry_size;
883 l1_num_ids = l1_num_ids / l2_num_ids;
884 l1_entry_size = GITS_INDIRECT_ENTRY_SIZE;
885 }
886 table_size = roundup2(l1_entry_size * l1_num_ids, page_size);
887 if (howmany(table_size, page_size) > GITS_BASER_Size + 1) {
888 DPRINTF(("ITS: clamp table size 0x%lx -> ", table_size));
889 table_size = (GITS_BASER_Size + 1) * page_size;
890 DPRINTF(("0x%lx\n", table_size));
891 }
892 table_type = "Devices";
893
894 DPRINTF(("ITS: table_size is 0x%lx\n", table_size));
895
896 itstab = &its->its_tab_device;
897 itstab->tab_page_size = page_size;
898 itstab->tab_l1_entry_size = l1_entry_size;
899 itstab->tab_l1_num_ids = l1_num_ids;
900 itstab->tab_l2_entry_size = l2_entry_size;
901 itstab->tab_l2_num_ids = l2_num_ids;
902 itstab->tab_indirect = indirect;
903 LIST_INIT(&itstab->tab_pt);
904 break;
905 case GITS_Type_InterruptCollections:
906 /*
907 * Allocate space for one interrupt collection per CPU.
908 */
909 table_size = roundup(l1_entry_size * ncpu, page_size);
910 table_type = "Collections";
911 break;
912 default:
913 table_size = 0;
914 break;
915 }
916
917 if (table_size == 0)
918 continue;
919
920 gicv3_dma_alloc(sc, &its->its_tab[tab], table_size, table_align);
921 if (its->its_cmd_flush) {
922 cpu_dcache_wb_range((vaddr_t)its->its_tab[tab].base, table_size);
923 }
924 dsb(sy);
925
926 baser &= ~GITS_BASER_Size;
927 baser |= __SHIFTIN(howmany(table_size, page_size) - 1, GITS_BASER_Size);
928 baser &= ~GITS_BASER_Physical_Address;
929 baser |= its->its_tab[tab].segs[0].ds_addr;
930 baser &= ~GITS_BASER_InnerCache;
931 baser |= __SHIFTIN(innercache, GITS_BASER_InnerCache);
932 baser &= ~GITS_BASER_Shareability;
933 baser |= __SHIFTIN(share, GITS_BASER_Shareability);
934 baser |= GITS_BASER_Valid;
935 if (indirect) {
936 baser |= GITS_BASER_Indirect;
937 } else {
938 baser &= ~GITS_BASER_Indirect;
939 }
940
941 gits_write_8(its, GITS_BASERn(tab), baser);
942
943 baser = gits_read_8(its, GITS_BASERn(tab));
944 if (__SHIFTOUT(baser, GITS_BASER_Shareability) == GITS_Shareability_NS) {
945 baser &= ~GITS_BASER_InnerCache;
946 baser |= __SHIFTIN(GITS_Cache_NORMAL_NC, GITS_BASER_InnerCache);
947
948 gits_write_8(its, GITS_BASERn(tab), baser);
949 }
950
951 baser = gits_read_8(its, GITS_BASERn(tab));
952 aprint_normal_dev(sc->sc_dev, "ITS [#%d] %s table @ %#lx/%#lx, %s, %s%s\n",
953 tab, table_type, its->its_tab[tab].segs[0].ds_addr, table_size,
954 gits_cache_type[__SHIFTOUT(baser, GITS_BASER_InnerCache)],
955 gits_share_type[__SHIFTOUT(baser, GITS_BASER_Shareability)],
956 indirect ? ", indirect" : "");
957
958 if (__SHIFTOUT(baser, GITS_BASER_Type) == GITS_Type_Devices) {
959 its->its_tab_device.tab_l1 = its->its_tab[tab].base;
960 its->its_tab_device.tab_shareable =
961 __SHIFTOUT(baser, GITS_BASER_Shareability) != GITS_Shareability_NS;
962 }
963
964 }
965 }
966
967 static void
gicv3_its_enable(struct gicv3_softc * sc,struct gicv3_its * its)968 gicv3_its_enable(struct gicv3_softc *sc, struct gicv3_its *its)
969 {
970 uint32_t ctlr;
971
972 ctlr = gits_read_4(its, GITS_CTLR);
973 ctlr |= GITS_CTLR_Enabled;
974 gits_write_4(its, GITS_CTLR, ctlr);
975 }
976
977 static void
gicv3_its_cpu_init(void * priv,struct cpu_info * ci)978 gicv3_its_cpu_init(void *priv, struct cpu_info *ci)
979 {
980 struct gicv3_its * const its = priv;
981 struct gicv3_softc * const sc = its->its_gic;
982 uint64_t rdbase;
983 size_t irq;
984
985 const uint64_t typer = bus_space_read_8(sc->sc_bst, its->its_bsh, GITS_TYPER);
986 if (typer & GITS_TYPER_PTA) {
987 void *va = bus_space_vaddr(sc->sc_bst, sc->sc_bsh_r[ci->ci_gic_redist]);
988 rdbase = vtophys((vaddr_t)va);
989 } else {
990 rdbase = (uint64_t)sc->sc_processor_id[cpu_index(ci)] << 16;
991 }
992 its->its_rdbase[cpu_index(ci)] = rdbase;
993
994 /*
995 * Map collection ID of this CPU's index to this CPU's redistributor.
996 */
997 mutex_enter(its->its_lock);
998 gits_command_mapc(its, cpu_index(ci), rdbase, true);
999 gits_command_invall(its, cpu_index(ci));
1000 gits_wait(its);
1001
1002 /*
1003 * Update routing for LPIs targetting this CPU
1004 */
1005 for (irq = 0; irq < its->its_pic->pic_maxsources; irq++) {
1006 if (its->its_targets[irq] != ci)
1007 continue;
1008 KASSERT(its->its_pa[irq] != NULL);
1009
1010 const uint32_t devid = its->its_devid[irq];
1011 gits_command_movi(its, devid, irq, cpu_index(ci));
1012 gits_command_sync(its, its->its_rdbase[cpu_index(ci)]);
1013 }
1014 gits_wait(its);
1015 mutex_exit(its->its_lock);
1016
1017 its->its_cpuonline[cpu_index(ci)] = true;
1018 }
1019
1020 static void
gicv3_its_get_affinity(void * priv,size_t irq,kcpuset_t * affinity)1021 gicv3_its_get_affinity(void *priv, size_t irq, kcpuset_t *affinity)
1022 {
1023 struct gicv3_its * const its = priv;
1024 struct cpu_info *ci;
1025
1026 ci = its->its_targets[irq];
1027 if (ci)
1028 kcpuset_set(affinity, cpu_index(ci));
1029 }
1030
1031 static int
gicv3_its_set_affinity(void * priv,size_t irq,const kcpuset_t * affinity)1032 gicv3_its_set_affinity(void *priv, size_t irq, const kcpuset_t *affinity)
1033 {
1034 struct gicv3_its * const its = priv;
1035 const struct pci_attach_args *pa;
1036 struct cpu_info *ci;
1037
1038 const int set = kcpuset_countset(affinity);
1039 if (set != 1)
1040 return EINVAL;
1041
1042 pa = its->its_pa[irq];
1043 if (pa == NULL)
1044 return EPASSTHROUGH;
1045
1046 ci = cpu_lookup(kcpuset_ffs(affinity) - 1);
1047 its->its_targets[irq] = ci;
1048
1049 if (its->its_cpuonline[cpu_index(ci)] == true) {
1050 const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag);
1051 mutex_enter(its->its_lock);
1052 gits_command_movi(its, devid, irq, cpu_index(ci));
1053 gits_command_sync(its, its->its_rdbase[cpu_index(ci)]);
1054 mutex_exit(its->its_lock);
1055 }
1056
1057 return 0;
1058 }
1059
1060 int
gicv3_its_init(struct gicv3_softc * sc,bus_space_handle_t bsh,uint64_t its_base,uint32_t its_id)1061 gicv3_its_init(struct gicv3_softc *sc, bus_space_handle_t bsh,
1062 uint64_t its_base, uint32_t its_id)
1063 {
1064 struct gicv3_its *its;
1065 struct arm_pci_msi *msi;
1066
1067 const uint64_t typer = bus_space_read_8(sc->sc_bst, bsh, GITS_TYPER);
1068 if ((typer & GITS_TYPER_Physical) == 0)
1069 return ENXIO;
1070
1071 its = kmem_zalloc(sizeof(*its), KM_SLEEP);
1072 its->its_id = its_id;
1073 its->its_bst = sc->sc_bst;
1074 its->its_bsh = bsh;
1075 its->its_dmat = sc->sc_dmat;
1076 its->its_base = its_base;
1077 its->its_pic = &sc->sc_lpi;
1078 snprintf(its->its_pic->pic_name, sizeof(its->its_pic->pic_name), "gicv3-its");
1079 KASSERT(its->its_pic->pic_maxsources > 0);
1080 its->its_pa = kmem_zalloc(sizeof(struct pci_attach_args *) * its->its_pic->pic_maxsources, KM_SLEEP);
1081 its->its_targets = kmem_zalloc(sizeof(struct cpu_info *) * its->its_pic->pic_maxsources, KM_SLEEP);
1082 its->its_devid = kmem_zalloc(sizeof(uint32_t) * its->its_pic->pic_maxsources, KM_SLEEP);
1083 its->its_gic = sc;
1084 its->its_rdbase = kmem_zalloc(sizeof(*its->its_rdbase) * ncpu, KM_SLEEP);
1085 its->its_cpuonline = kmem_zalloc(sizeof(*its->its_cpuonline) * ncpu, KM_SLEEP);
1086 its->its_cb.cpu_init = gicv3_its_cpu_init;
1087 its->its_cb.get_affinity = gicv3_its_get_affinity;
1088 its->its_cb.set_affinity = gicv3_its_set_affinity;
1089 its->its_cb.priv = its;
1090 LIST_INIT(&its->its_devices);
1091 LIST_INSERT_HEAD(&sc->sc_lpi_callbacks, &its->its_cb, list);
1092 its->its_lock = mutex_obj_alloc(MUTEX_SPIN, IPL_NONE);
1093
1094 gicv3_its_command_init(sc, its);
1095 gicv3_its_table_init(sc, its);
1096
1097 gicv3_its_enable(sc, its);
1098
1099 gicv3_its_cpu_init(its, curcpu());
1100
1101 msi = &its->its_msi;
1102 msi->msi_id = its_id;
1103 msi->msi_dev = sc->sc_dev;
1104 msi->msi_priv = its;
1105 msi->msi_alloc = gicv3_its_msi_alloc;
1106 msi->msix_alloc = gicv3_its_msix_alloc;
1107 msi->msi_intr_establish = gicv3_its_msi_intr_establish;
1108 msi->msi_intr_release = gicv3_its_msi_intr_release;
1109
1110 return arm_pci_msi_add(msi);
1111 }
1112