1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright © 2021-2022 Dmitry Salychev
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #include <sys/cdefs.h>
29 /*
30 * QBMan command interface and the DPAA2 I/O (DPIO) driver.
31 *
32 * The DPIO object allows configuration of the QBMan software portal with
33 * optional notification capabilities.
34 *
35 * Software portals are used by the driver to communicate with the QBMan. The
36 * DPIO object’s main purpose is to enable the driver to perform I/O – enqueue
37 * and dequeue operations, as well as buffer release and acquire operations –
38 * using QBMan.
39 */
40
41 #include <sys/param.h>
42 #include <sys/kernel.h>
43 #include <sys/bus.h>
44 #include <sys/rman.h>
45 #include <sys/module.h>
46 #include <sys/malloc.h>
47 #include <sys/lock.h>
48 #include <sys/mutex.h>
49 #include <sys/_cpuset.h>
50 #include <sys/cpuset.h>
51 #include <sys/taskqueue.h>
52 #include <sys/smp.h>
53
54 #include <vm/vm.h>
55
56 #include <machine/bus.h>
57 #include <machine/resource.h>
58
59 #include <dev/pci/pcivar.h>
60
61 #include "pcib_if.h"
62 #include "pci_if.h"
63
64 #include "dpaa2_mc.h"
65 #include "dpaa2_mcp.h"
66 #include "dpaa2_swp.h"
67 #include "dpaa2_swp_if.h"
68 #include "dpaa2_cmd_if.h"
69 #include "dpaa2_io.h"
70 #include "dpaa2_ni.h"
71
72 #define DPIO_IRQ_INDEX 0 /* index of the only DPIO IRQ */
73 #define DPIO_POLL_MAX 32
74
75 /*
76 * Memory:
77 * 0: cache-enabled part of the QBMan software portal.
78 * 1: cache-inhibited part of the QBMan software portal.
79 * 2: control registers of the QBMan software portal?
80 *
81 * Note that MSI should be allocated separately using pseudo-PCI interface.
82 */
83 struct resource_spec dpaa2_io_spec[] = {
84 /*
85 * System Memory resources.
86 */
87 #define MEM_RES_NUM (3u)
88 #define MEM_RID_OFF (0u)
89 #define MEM_RID(rid) ((rid) + MEM_RID_OFF)
90 { SYS_RES_MEMORY, MEM_RID(0), RF_ACTIVE | RF_UNMAPPED },
91 { SYS_RES_MEMORY, MEM_RID(1), RF_ACTIVE | RF_UNMAPPED },
92 { SYS_RES_MEMORY, MEM_RID(2), RF_ACTIVE | RF_UNMAPPED | RF_OPTIONAL },
93 /*
94 * DPMCP resources.
95 *
96 * NOTE: MC command portals (MCPs) are used to send commands to, and
97 * receive responses from, the MC firmware. One portal per DPIO.
98 */
99 #define MCP_RES_NUM (1u)
100 #define MCP_RID_OFF (MEM_RID_OFF + MEM_RES_NUM)
101 #define MCP_RID(rid) ((rid) + MCP_RID_OFF)
102 /* --- */
103 { DPAA2_DEV_MCP, MCP_RID(0), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
104 /* --- */
105 RESOURCE_SPEC_END
106 };
107
108 /* Configuration routines. */
109 static int dpaa2_io_setup_irqs(device_t dev);
110 static int dpaa2_io_release_irqs(device_t dev);
111 static int dpaa2_io_setup_msi(struct dpaa2_io_softc *sc);
112 static int dpaa2_io_release_msi(struct dpaa2_io_softc *sc);
113
114 /* Interrupt handlers */
115 static void dpaa2_io_intr(void *arg);
116
117 static int
dpaa2_io_probe(device_t dev)118 dpaa2_io_probe(device_t dev)
119 {
120 /* DPIO device will be added by a parent resource container itself. */
121 device_set_desc(dev, "DPAA2 I/O");
122 return (BUS_PROBE_DEFAULT);
123 }
124
125 static int
dpaa2_io_detach(device_t dev)126 dpaa2_io_detach(device_t dev)
127 {
128 device_t child = dev;
129 struct dpaa2_io_softc *sc = device_get_softc(dev);
130 struct dpaa2_devinfo *dinfo = device_get_ivars(dev);
131 int error;
132
133 /* Tear down interrupt handler and release IRQ resources. */
134 dpaa2_io_release_irqs(dev);
135
136 /* Free software portal helper object. */
137 dpaa2_swp_free_portal(sc->swp);
138
139 /* Disable DPIO object. */
140 error = DPAA2_CMD_IO_DISABLE(dev, child, dpaa2_mcp_tk(sc->cmd,
141 sc->io_token));
142 if (error && bootverbose)
143 device_printf(dev, "%s: failed to disable DPIO: id=%d, "
144 "error=%d\n", __func__, dinfo->id, error);
145
146 /* Close control sessions with the DPAA2 objects. */
147 DPAA2_CMD_IO_CLOSE(dev, child, dpaa2_mcp_tk(sc->cmd, sc->io_token));
148 DPAA2_CMD_RC_CLOSE(dev, child, dpaa2_mcp_tk(sc->cmd, sc->rc_token));
149
150 /* Free pre-allocated MC command. */
151 dpaa2_mcp_free_command(sc->cmd);
152 sc->cmd = NULL;
153 sc->io_token = 0;
154 sc->rc_token = 0;
155
156 /* Unmap memory resources of the portal. */
157 for (int i = 0; i < MEM_RES_NUM; i++) {
158 if (sc->res[MEM_RID(i)] == NULL)
159 continue;
160 error = bus_unmap_resource(sc->dev, SYS_RES_MEMORY,
161 sc->res[MEM_RID(i)], &sc->map[MEM_RID(i)]);
162 if (error && bootverbose)
163 device_printf(dev, "%s: failed to unmap memory "
164 "resource: rid=%d, error=%d\n", __func__, MEM_RID(i),
165 error);
166 }
167
168 /* Release allocated resources. */
169 bus_release_resources(dev, dpaa2_io_spec, sc->res);
170
171 return (0);
172 }
173
174 static int
dpaa2_io_attach(device_t dev)175 dpaa2_io_attach(device_t dev)
176 {
177 device_t pdev = device_get_parent(dev);
178 device_t child = dev;
179 device_t mcp_dev;
180 struct dpaa2_io_softc *sc = device_get_softc(dev);
181 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev);
182 struct dpaa2_devinfo *dinfo = device_get_ivars(dev);
183 struct dpaa2_devinfo *mcp_dinfo;
184 struct resource_map_request req;
185 struct {
186 vm_memattr_t memattr;
187 char *label;
188 } map_args[MEM_RES_NUM] = {
189 { VM_MEMATTR_WRITE_BACK, "cache-enabled part" },
190 { VM_MEMATTR_DEVICE, "cache-inhibited part" },
191 { VM_MEMATTR_DEVICE, "control registers" }
192 };
193 int error;
194
195 sc->dev = dev;
196 sc->swp = NULL;
197 sc->cmd = NULL;
198 sc->intr = NULL;
199 sc->irq_resource = NULL;
200
201 /* Allocate resources. */
202 error = bus_alloc_resources(sc->dev, dpaa2_io_spec, sc->res);
203 if (error) {
204 device_printf(dev, "%s: failed to allocate resources: "
205 "error=%d\n", __func__, error);
206 return (ENXIO);
207 }
208
209 /* Set allocated MC portal up. */
210 mcp_dev = (device_t) rman_get_start(sc->res[MCP_RID(0)]);
211 mcp_dinfo = device_get_ivars(mcp_dev);
212 dinfo->portal = mcp_dinfo->portal;
213
214 /* Map memory resources of the portal. */
215 for (int i = 0; i < MEM_RES_NUM; i++) {
216 if (sc->res[MEM_RID(i)] == NULL)
217 continue;
218
219 resource_init_map_request(&req);
220 req.memattr = map_args[i].memattr;
221 error = bus_map_resource(sc->dev, SYS_RES_MEMORY,
222 sc->res[MEM_RID(i)], &req, &sc->map[MEM_RID(i)]);
223 if (error) {
224 device_printf(dev, "%s: failed to map %s: error=%d\n",
225 __func__, map_args[i].label, error);
226 goto err_exit;
227 }
228 }
229
230 /* Allocate a command to send to the MC hardware. */
231 error = dpaa2_mcp_init_command(&sc->cmd, DPAA2_CMD_DEF);
232 if (error) {
233 device_printf(dev, "%s: failed to allocate dpaa2_cmd: "
234 "error=%d\n", __func__, error);
235 goto err_exit;
236 }
237
238 /* Prepare DPIO object. */
239 error = DPAA2_CMD_RC_OPEN(dev, child, sc->cmd, rcinfo->id,
240 &sc->rc_token);
241 if (error) {
242 device_printf(dev, "%s: failed to open DPRC: error=%d\n",
243 __func__, error);
244 goto err_exit;
245 }
246 error = DPAA2_CMD_IO_OPEN(dev, child, sc->cmd, dinfo->id, &sc->io_token);
247 if (error) {
248 device_printf(dev, "%s: failed to open DPIO: id=%d, error=%d\n",
249 __func__, dinfo->id, error);
250 goto err_exit;
251 }
252 error = DPAA2_CMD_IO_RESET(dev, child, sc->cmd);
253 if (error) {
254 device_printf(dev, "%s: failed to reset DPIO: id=%d, error=%d\n",
255 __func__, dinfo->id, error);
256 goto err_exit;
257 }
258 error = DPAA2_CMD_IO_GET_ATTRIBUTES(dev, child, sc->cmd, &sc->attr);
259 if (error) {
260 device_printf(dev, "%s: failed to get DPIO attributes: id=%d, "
261 "error=%d\n", __func__, dinfo->id, error);
262 goto err_exit;
263 }
264 error = DPAA2_CMD_IO_ENABLE(dev, child, sc->cmd);
265 if (error) {
266 device_printf(dev, "%s: failed to enable DPIO: id=%d, "
267 "error=%d\n", __func__, dinfo->id, error);
268 goto err_exit;
269 }
270
271 /* Prepare descriptor of the QBMan software portal. */
272 sc->swp_desc.dpio_dev = dev;
273 sc->swp_desc.swp_version = sc->attr.swp_version;
274 sc->swp_desc.swp_clk = sc->attr.swp_clk;
275 sc->swp_desc.swp_id = sc->attr.swp_id;
276 sc->swp_desc.has_notif = sc->attr.priors_num ? true : false;
277 sc->swp_desc.has_8prio = sc->attr.priors_num == 8u ? true : false;
278
279 sc->swp_desc.cena_res = sc->res[0];
280 sc->swp_desc.cena_map = &sc->map[0];
281 sc->swp_desc.cinh_res = sc->res[1];
282 sc->swp_desc.cinh_map = &sc->map[1];
283
284 /*
285 * Compute how many 256 QBMAN cycles fit into one ns. This is because
286 * the interrupt timeout period register needs to be specified in QBMAN
287 * clock cycles in increments of 256.
288 */
289 sc->swp_desc.swp_cycles_ratio = 256000 /
290 (sc->swp_desc.swp_clk / 1000000);
291
292 /* Initialize QBMan software portal. */
293 error = dpaa2_swp_init_portal(&sc->swp, &sc->swp_desc, DPAA2_SWP_DEF);
294 if (error) {
295 device_printf(dev, "%s: failed to initialize dpaa2_swp: "
296 "error=%d\n", __func__, error);
297 goto err_exit;
298 }
299
300 error = dpaa2_io_setup_irqs(dev);
301 if (error) {
302 device_printf(dev, "%s: failed to setup IRQs: error=%d\n",
303 __func__, error);
304 goto err_exit;
305 }
306
307 #if 0
308 /* TODO: Enable debug output via sysctl (to reduce output). */
309 if (bootverbose)
310 device_printf(dev, "dpio_id=%d, swp_id=%d, chan_mode=%s, "
311 "notif_priors=%d, swp_version=0x%x\n",
312 sc->attr.id, sc->attr.swp_id,
313 sc->attr.chan_mode == DPAA2_IO_LOCAL_CHANNEL
314 ? "local_channel" : "no_channel", sc->attr.priors_num,
315 sc->attr.swp_version);
316 #endif
317 return (0);
318
319 err_exit:
320 dpaa2_io_detach(dev);
321 return (ENXIO);
322 }
323
324 /**
325 * @brief Enqueue multiple frames to a frame queue using one FQID.
326 */
327 static int
dpaa2_io_enq_multiple_fq(device_t iodev,uint32_t fqid,struct dpaa2_fd * fd,int frames_n)328 dpaa2_io_enq_multiple_fq(device_t iodev, uint32_t fqid,
329 struct dpaa2_fd *fd, int frames_n)
330 {
331 struct dpaa2_io_softc *sc = device_get_softc(iodev);
332 struct dpaa2_swp *swp = sc->swp;
333 struct dpaa2_eq_desc ed;
334 uint32_t flags = 0;
335
336 memset(&ed, 0, sizeof(ed));
337
338 /* Setup enqueue descriptor. */
339 dpaa2_swp_set_ed_norp(&ed, false);
340 dpaa2_swp_set_ed_fq(&ed, fqid);
341
342 return (dpaa2_swp_enq_mult(swp, &ed, fd, &flags, frames_n));
343 }
344
345 /**
346 * @brief Configure the channel data availability notification (CDAN)
347 * in a particular WQ channel paired with DPIO.
348 */
349 static int
dpaa2_io_conf_wq_channel(device_t iodev,struct dpaa2_io_notif_ctx * ctx)350 dpaa2_io_conf_wq_channel(device_t iodev, struct dpaa2_io_notif_ctx *ctx)
351 {
352 struct dpaa2_io_softc *sc = device_get_softc(iodev);
353
354 /* Enable generation of the CDAN notifications. */
355 if (ctx->cdan_en)
356 return (dpaa2_swp_conf_wq_channel(sc->swp, ctx->fq_chan_id,
357 DPAA2_WQCHAN_WE_EN | DPAA2_WQCHAN_WE_CTX, ctx->cdan_en,
358 ctx->qman_ctx));
359
360 return (0);
361 }
362
363 /**
364 * @brief Query current configuration/state of the buffer pool.
365 */
366 static int
dpaa2_io_query_bp(device_t iodev,uint16_t bpid,struct dpaa2_bp_conf * conf)367 dpaa2_io_query_bp(device_t iodev, uint16_t bpid, struct dpaa2_bp_conf *conf)
368 {
369 struct dpaa2_io_softc *sc = device_get_softc(iodev);
370
371 return (dpaa2_swp_query_bp(sc->swp, bpid, conf));
372 }
373
374 /**
375 * @brief Release one or more buffer pointers to the QBMan buffer pool.
376 */
377 static int
dpaa2_io_release_bufs(device_t iodev,uint16_t bpid,bus_addr_t * buf,uint32_t buf_num)378 dpaa2_io_release_bufs(device_t iodev, uint16_t bpid, bus_addr_t *buf,
379 uint32_t buf_num)
380 {
381 struct dpaa2_io_softc *sc = device_get_softc(iodev);
382
383 return (dpaa2_swp_release_bufs(sc->swp, bpid, buf, buf_num));
384 }
385
386 /**
387 * @brief Configure DPNI object to generate interrupts.
388 */
389 static int
dpaa2_io_setup_irqs(device_t dev)390 dpaa2_io_setup_irqs(device_t dev)
391 {
392 struct dpaa2_io_softc *sc = device_get_softc(dev);
393 int error;
394
395 /*
396 * Setup interrupts generated by the software portal.
397 */
398 dpaa2_swp_set_intr_trigger(sc->swp, DPAA2_SWP_INTR_DQRI);
399 dpaa2_swp_clear_intr_status(sc->swp, 0xFFFFFFFFu);
400
401 /* Configure IRQs. */
402 error = dpaa2_io_setup_msi(sc);
403 if (error) {
404 device_printf(dev, "%s: failed to allocate MSI: error=%d\n",
405 __func__, error);
406 return (error);
407 }
408 if ((sc->irq_resource = bus_alloc_resource_any(dev, SYS_RES_IRQ,
409 &sc->irq_rid[0], RF_ACTIVE | RF_SHAREABLE)) == NULL) {
410 device_printf(dev, "%s: failed to allocate IRQ resource\n",
411 __func__);
412 return (ENXIO);
413 }
414 if (bus_setup_intr(dev, sc->irq_resource, INTR_TYPE_NET | INTR_MPSAFE |
415 INTR_ENTROPY, NULL, dpaa2_io_intr, sc, &sc->intr)) {
416 device_printf(dev, "%s: failed to setup IRQ resource\n",
417 __func__);
418 return (ENXIO);
419 }
420
421 /* Wrap DPIO ID around number of CPUs. */
422 bus_bind_intr(dev, sc->irq_resource, sc->attr.id % mp_ncpus);
423
424 /*
425 * Setup and enable Static Dequeue Command to receive CDANs from
426 * channel 0.
427 */
428 if (sc->swp_desc.has_notif)
429 dpaa2_swp_set_push_dequeue(sc->swp, 0, true);
430
431 return (0);
432 }
433
434 static int
dpaa2_io_release_irqs(device_t dev)435 dpaa2_io_release_irqs(device_t dev)
436 {
437 struct dpaa2_io_softc *sc = device_get_softc(dev);
438
439 /* Disable receiving CDANs from channel 0. */
440 if (sc->swp_desc.has_notif)
441 dpaa2_swp_set_push_dequeue(sc->swp, 0, false);
442
443 /* Release IRQ resources. */
444 if (sc->intr != NULL)
445 bus_teardown_intr(dev, sc->irq_resource, &sc->intr);
446 if (sc->irq_resource != NULL)
447 bus_release_resource(dev, SYS_RES_IRQ, sc->irq_rid[0],
448 sc->irq_resource);
449
450 (void)dpaa2_io_release_msi(device_get_softc(dev));
451
452 /* Configure software portal to stop generating interrupts. */
453 dpaa2_swp_set_intr_trigger(sc->swp, 0);
454 dpaa2_swp_clear_intr_status(sc->swp, 0xFFFFFFFFu);
455
456 return (0);
457 }
458
459 /**
460 * @brief Allocate MSI interrupts for this DPAA2 I/O object.
461 */
462 static int
dpaa2_io_setup_msi(struct dpaa2_io_softc * sc)463 dpaa2_io_setup_msi(struct dpaa2_io_softc *sc)
464 {
465 int val;
466
467 val = pci_msi_count(sc->dev);
468 if (val < DPAA2_IO_MSI_COUNT)
469 device_printf(sc->dev, "MSI: actual=%d, expected=%d\n", val,
470 DPAA2_IO_MSI_COUNT);
471 val = MIN(val, DPAA2_IO_MSI_COUNT);
472
473 if (pci_alloc_msi(sc->dev, &val) != 0)
474 return (EINVAL);
475
476 for (int i = 0; i < val; i++)
477 sc->irq_rid[i] = i + 1;
478
479 return (0);
480 }
481
482 static int
dpaa2_io_release_msi(struct dpaa2_io_softc * sc)483 dpaa2_io_release_msi(struct dpaa2_io_softc *sc)
484 {
485 int error;
486
487 error = pci_release_msi(sc->dev);
488 if (error) {
489 device_printf(sc->dev, "%s: failed to release MSI: error=%d/n",
490 __func__, error);
491 return (error);
492 }
493
494 return (0);
495 }
496
497 /**
498 * @brief DPAA2 I/O interrupt handler.
499 */
500 static void
dpaa2_io_intr(void * arg)501 dpaa2_io_intr(void *arg)
502 {
503 struct dpaa2_io_softc *sc = (struct dpaa2_io_softc *) arg;
504 struct dpaa2_io_notif_ctx *ctx[DPIO_POLL_MAX];
505 struct dpaa2_dq dq;
506 uint32_t idx, status;
507 uint16_t flags;
508 int rc, cdan_n = 0;
509
510 status = dpaa2_swp_read_intr_status(sc->swp);
511 if (status == 0) {
512 return;
513 }
514
515 DPAA2_SWP_LOCK(sc->swp, &flags);
516 if (flags & DPAA2_SWP_DESTROYED) {
517 /* Terminate operation if portal is destroyed. */
518 DPAA2_SWP_UNLOCK(sc->swp);
519 return;
520 }
521
522 for (int i = 0; i < DPIO_POLL_MAX; i++) {
523 rc = dpaa2_swp_dqrr_next_locked(sc->swp, &dq, &idx);
524 if (rc) {
525 break;
526 }
527
528 if ((dq.common.verb & DPAA2_DQRR_RESULT_MASK) ==
529 DPAA2_DQRR_RESULT_CDAN) {
530 ctx[cdan_n++] = (struct dpaa2_io_notif_ctx *) dq.scn.ctx;
531 } else {
532 /* TODO: Report unknown DQRR entry. */
533 }
534 dpaa2_swp_write_reg(sc->swp, DPAA2_SWP_CINH_DCAP, idx);
535 }
536 DPAA2_SWP_UNLOCK(sc->swp);
537
538 for (int i = 0; i < cdan_n; i++) {
539 ctx[i]->poll(ctx[i]->channel);
540 }
541
542 /* Enable software portal interrupts back */
543 dpaa2_swp_clear_intr_status(sc->swp, status);
544 dpaa2_swp_write_reg(sc->swp, DPAA2_SWP_CINH_IIR, 0);
545 }
546
547 static device_method_t dpaa2_io_methods[] = {
548 /* Device interface */
549 DEVMETHOD(device_probe, dpaa2_io_probe),
550 DEVMETHOD(device_attach, dpaa2_io_attach),
551 DEVMETHOD(device_detach, dpaa2_io_detach),
552
553 /* QBMan software portal interface */
554 DEVMETHOD(dpaa2_swp_enq_multiple_fq, dpaa2_io_enq_multiple_fq),
555 DEVMETHOD(dpaa2_swp_conf_wq_channel, dpaa2_io_conf_wq_channel),
556 DEVMETHOD(dpaa2_swp_query_bp, dpaa2_io_query_bp),
557 DEVMETHOD(dpaa2_swp_release_bufs, dpaa2_io_release_bufs),
558
559 DEVMETHOD_END
560 };
561
562 static driver_t dpaa2_io_driver = {
563 "dpaa2_io",
564 dpaa2_io_methods,
565 sizeof(struct dpaa2_io_softc),
566 };
567
568 DRIVER_MODULE(dpaa2_io, dpaa2_rc, dpaa2_io_driver, 0, 0);
569