1 /*
2 * Copyright (C) 2015 Cavium Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 */
27 #include <sys/cdefs.h>
28 #include "opt_inet.h"
29 #include "opt_inet6.h"
30
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/bitset.h>
34 #include <sys/bitstring.h>
35 #include <sys/buf_ring.h>
36 #include <sys/bus.h>
37 #include <sys/endian.h>
38 #include <sys/kernel.h>
39 #include <sys/malloc.h>
40 #include <sys/module.h>
41 #include <sys/rman.h>
42 #include <sys/pciio.h>
43 #include <sys/pcpu.h>
44 #include <sys/proc.h>
45 #include <sys/sockio.h>
46 #include <sys/socket.h>
47 #include <sys/stdatomic.h>
48 #include <sys/cpuset.h>
49 #include <sys/lock.h>
50 #include <sys/mutex.h>
51 #include <sys/smp.h>
52 #include <sys/taskqueue.h>
53
54 #include <vm/vm.h>
55 #include <vm/pmap.h>
56
57 #include <machine/bus.h>
58 #include <machine/vmparam.h>
59
60 #include <net/if.h>
61 #include <net/if_var.h>
62 #include <net/if_media.h>
63 #include <net/ifq.h>
64 #include <net/bpf.h>
65 #include <net/ethernet.h>
66
67 #include <netinet/in_systm.h>
68 #include <netinet/in.h>
69 #include <netinet/if_ether.h>
70 #include <netinet/ip.h>
71 #include <netinet/ip6.h>
72 #include <netinet/sctp.h>
73 #include <netinet/tcp.h>
74 #include <netinet/tcp_lro.h>
75 #include <netinet/udp.h>
76
77 #include <netinet6/ip6_var.h>
78
79 #include <dev/pci/pcireg.h>
80 #include <dev/pci/pcivar.h>
81
82 #include "thunder_bgx.h"
83 #include "nic_reg.h"
84 #include "nic.h"
85 #include "q_struct.h"
86 #include "nicvf_queues.h"
87
88 #define DEBUG
89 #undef DEBUG
90
91 #ifdef DEBUG
92 #define dprintf(dev, fmt, ...) device_printf(dev, fmt, ##__VA_ARGS__)
93 #else
94 #define dprintf(dev, fmt, ...)
95 #endif
96
97 MALLOC_DECLARE(M_NICVF);
98
99 static void nicvf_free_snd_queue(struct nicvf *, struct snd_queue *);
100 static struct mbuf * nicvf_get_rcv_mbuf(struct nicvf *, struct cqe_rx_t *);
101 static void nicvf_sq_disable(struct nicvf *, int);
102 static void nicvf_sq_enable(struct nicvf *, struct snd_queue *, int);
103 static void nicvf_put_sq_desc(struct snd_queue *, int);
104 static void nicvf_cmp_queue_config(struct nicvf *, struct queue_set *, int,
105 boolean_t);
106 static void nicvf_sq_free_used_descs(struct nicvf *, struct snd_queue *, int);
107
108 static int nicvf_tx_mbuf_locked(struct snd_queue *, struct mbuf **);
109
110 static void nicvf_rbdr_task(void *, int);
111 static void nicvf_rbdr_task_nowait(void *, int);
112
113 struct rbuf_info {
114 bus_dma_tag_t dmat;
115 bus_dmamap_t dmap;
116 struct mbuf * mbuf;
117 };
118
119 #define GET_RBUF_INFO(x) ((struct rbuf_info *)((x) - NICVF_RCV_BUF_ALIGN_BYTES))
120
121 /* Poll a register for a specific value */
nicvf_poll_reg(struct nicvf * nic,int qidx,uint64_t reg,int bit_pos,int bits,int val)122 static int nicvf_poll_reg(struct nicvf *nic, int qidx,
123 uint64_t reg, int bit_pos, int bits, int val)
124 {
125 uint64_t bit_mask;
126 uint64_t reg_val;
127 int timeout = 10;
128
129 bit_mask = (1UL << bits) - 1;
130 bit_mask = (bit_mask << bit_pos);
131
132 while (timeout) {
133 reg_val = nicvf_queue_reg_read(nic, reg, qidx);
134 if (((reg_val & bit_mask) >> bit_pos) == val)
135 return (0);
136
137 DELAY(1000);
138 timeout--;
139 }
140 device_printf(nic->dev, "Poll on reg 0x%lx failed\n", reg);
141 return (ETIMEDOUT);
142 }
143
144 /* Callback for bus_dmamap_load() */
145 static void
nicvf_dmamap_q_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)146 nicvf_dmamap_q_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
147 {
148 bus_addr_t *paddr;
149
150 KASSERT(nseg == 1, ("wrong number of segments, should be 1"));
151 paddr = arg;
152 *paddr = segs->ds_addr;
153 }
154
155 /* Allocate memory for a queue's descriptors */
156 static int
nicvf_alloc_q_desc_mem(struct nicvf * nic,struct q_desc_mem * dmem,int q_len,int desc_size,int align_bytes)157 nicvf_alloc_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem,
158 int q_len, int desc_size, int align_bytes)
159 {
160 int err, err_dmat;
161
162 /* Create DMA tag first */
163 err = bus_dma_tag_create(
164 bus_get_dma_tag(nic->dev), /* parent tag */
165 align_bytes, /* alignment */
166 0, /* boundary */
167 BUS_SPACE_MAXADDR, /* lowaddr */
168 BUS_SPACE_MAXADDR, /* highaddr */
169 NULL, NULL, /* filtfunc, filtfuncarg */
170 (q_len * desc_size), /* maxsize */
171 1, /* nsegments */
172 (q_len * desc_size), /* maxsegsize */
173 0, /* flags */
174 NULL, NULL, /* lockfunc, lockfuncarg */
175 &dmem->dmat); /* dmat */
176
177 if (err != 0) {
178 device_printf(nic->dev,
179 "Failed to create busdma tag for descriptors ring\n");
180 return (err);
181 }
182
183 /* Allocate segment of continuous DMA safe memory */
184 err = bus_dmamem_alloc(
185 dmem->dmat, /* DMA tag */
186 &dmem->base, /* virtual address */
187 (BUS_DMA_NOWAIT | BUS_DMA_ZERO), /* flags */
188 &dmem->dmap); /* DMA map */
189 if (err != 0) {
190 device_printf(nic->dev, "Failed to allocate DMA safe memory for"
191 "descriptors ring\n");
192 goto dmamem_fail;
193 }
194
195 err = bus_dmamap_load(
196 dmem->dmat,
197 dmem->dmap,
198 dmem->base,
199 (q_len * desc_size), /* allocation size */
200 nicvf_dmamap_q_cb, /* map to DMA address cb. */
201 &dmem->phys_base, /* physical address */
202 BUS_DMA_NOWAIT);
203 if (err != 0) {
204 device_printf(nic->dev,
205 "Cannot load DMA map of descriptors ring\n");
206 goto dmamap_fail;
207 }
208
209 dmem->q_len = q_len;
210 dmem->size = (desc_size * q_len);
211
212 return (0);
213
214 dmamap_fail:
215 bus_dmamem_free(dmem->dmat, dmem->base, dmem->dmap);
216 dmem->phys_base = 0;
217 dmamem_fail:
218 err_dmat = bus_dma_tag_destroy(dmem->dmat);
219 dmem->base = NULL;
220 KASSERT(err_dmat == 0,
221 ("%s: Trying to destroy BUSY DMA tag", __func__));
222
223 return (err);
224 }
225
226 /* Free queue's descriptor memory */
227 static void
nicvf_free_q_desc_mem(struct nicvf * nic,struct q_desc_mem * dmem)228 nicvf_free_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem)
229 {
230 int err;
231
232 if ((dmem == NULL) || (dmem->base == NULL))
233 return;
234
235 /* Unload a map */
236 bus_dmamap_sync(dmem->dmat, dmem->dmap, BUS_DMASYNC_POSTREAD);
237 bus_dmamap_unload(dmem->dmat, dmem->dmap);
238 /* Free DMA memory */
239 bus_dmamem_free(dmem->dmat, dmem->base, dmem->dmap);
240 /* Destroy DMA tag */
241 err = bus_dma_tag_destroy(dmem->dmat);
242
243 KASSERT(err == 0,
244 ("%s: Trying to destroy BUSY DMA tag", __func__));
245
246 dmem->phys_base = 0;
247 dmem->base = NULL;
248 }
249
250 /*
251 * Allocate buffer for packet reception
252 * HW returns memory address where packet is DMA'ed but not a pointer
253 * into RBDR ring, so save buffer address at the start of fragment and
254 * align the start address to a cache aligned address
255 */
256 static __inline int
nicvf_alloc_rcv_buffer(struct nicvf * nic,struct rbdr * rbdr,bus_dmamap_t dmap,int mflags,uint32_t buf_len,bus_addr_t * rbuf)257 nicvf_alloc_rcv_buffer(struct nicvf *nic, struct rbdr *rbdr,
258 bus_dmamap_t dmap, int mflags, uint32_t buf_len, bus_addr_t *rbuf)
259 {
260 struct mbuf *mbuf;
261 struct rbuf_info *rinfo;
262 bus_dma_segment_t segs[1];
263 int nsegs;
264 int err;
265
266 mbuf = m_getjcl(mflags, MT_DATA, M_PKTHDR, MCLBYTES);
267 if (mbuf == NULL)
268 return (ENOMEM);
269
270 /*
271 * The length is equal to the actual length + one 128b line
272 * used as a room for rbuf_info structure.
273 */
274 mbuf->m_len = mbuf->m_pkthdr.len = buf_len;
275
276 err = bus_dmamap_load_mbuf_sg(rbdr->rbdr_buff_dmat, dmap, mbuf, segs,
277 &nsegs, BUS_DMA_NOWAIT);
278 if (err != 0) {
279 device_printf(nic->dev,
280 "Failed to map mbuf into DMA visible memory, err: %d\n",
281 err);
282 m_freem(mbuf);
283 bus_dmamap_destroy(rbdr->rbdr_buff_dmat, dmap);
284 return (err);
285 }
286 if (nsegs != 1)
287 panic("Unexpected number of DMA segments for RB: %d", nsegs);
288 /*
289 * Now use the room for rbuf_info structure
290 * and adjust mbuf data and length.
291 */
292 rinfo = (struct rbuf_info *)mbuf->m_data;
293 m_adj(mbuf, NICVF_RCV_BUF_ALIGN_BYTES);
294
295 rinfo->dmat = rbdr->rbdr_buff_dmat;
296 rinfo->dmap = dmap;
297 rinfo->mbuf = mbuf;
298
299 *rbuf = segs[0].ds_addr + NICVF_RCV_BUF_ALIGN_BYTES;
300
301 return (0);
302 }
303
304 /* Retrieve mbuf for received packet */
305 static struct mbuf *
nicvf_rb_ptr_to_mbuf(struct nicvf * nic,bus_addr_t rb_ptr)306 nicvf_rb_ptr_to_mbuf(struct nicvf *nic, bus_addr_t rb_ptr)
307 {
308 struct mbuf *mbuf;
309 struct rbuf_info *rinfo;
310
311 /* Get buffer start address and alignment offset */
312 rinfo = GET_RBUF_INFO(PHYS_TO_DMAP(rb_ptr));
313
314 /* Now retrieve mbuf to give to stack */
315 mbuf = rinfo->mbuf;
316 if (__predict_false(mbuf == NULL)) {
317 panic("%s: Received packet fragment with NULL mbuf",
318 device_get_nameunit(nic->dev));
319 }
320 /*
321 * Clear the mbuf in the descriptor to indicate
322 * that this slot is processed and free to use.
323 */
324 rinfo->mbuf = NULL;
325
326 bus_dmamap_sync(rinfo->dmat, rinfo->dmap, BUS_DMASYNC_POSTREAD);
327 bus_dmamap_unload(rinfo->dmat, rinfo->dmap);
328
329 return (mbuf);
330 }
331
332 /* Allocate RBDR ring and populate receive buffers */
333 static int
nicvf_init_rbdr(struct nicvf * nic,struct rbdr * rbdr,int ring_len,int buf_size,int qidx)334 nicvf_init_rbdr(struct nicvf *nic, struct rbdr *rbdr, int ring_len,
335 int buf_size, int qidx)
336 {
337 bus_dmamap_t dmap;
338 bus_addr_t rbuf;
339 struct rbdr_entry_t *desc;
340 int idx;
341 int err;
342
343 /* Allocate rbdr descriptors ring */
344 err = nicvf_alloc_q_desc_mem(nic, &rbdr->dmem, ring_len,
345 sizeof(struct rbdr_entry_t), NICVF_RCV_BUF_ALIGN_BYTES);
346 if (err != 0) {
347 device_printf(nic->dev,
348 "Failed to create RBDR descriptors ring\n");
349 return (err);
350 }
351
352 rbdr->desc = rbdr->dmem.base;
353 /*
354 * Buffer size has to be in multiples of 128 bytes.
355 * Make room for metadata of size of one line (128 bytes).
356 */
357 rbdr->dma_size = buf_size - NICVF_RCV_BUF_ALIGN_BYTES;
358 rbdr->enable = TRUE;
359 rbdr->thresh = RBDR_THRESH;
360 rbdr->nic = nic;
361 rbdr->idx = qidx;
362
363 /*
364 * Create DMA tag for Rx buffers.
365 * Each map created using this tag is intended to store Rx payload for
366 * one fragment and one header structure containing rbuf_info (thus
367 * additional 128 byte line since RB must be a multiple of 128 byte
368 * cache line).
369 */
370 if (buf_size > MCLBYTES) {
371 device_printf(nic->dev,
372 "Buffer size to large for mbuf cluster\n");
373 return (EINVAL);
374 }
375 err = bus_dma_tag_create(
376 bus_get_dma_tag(nic->dev), /* parent tag */
377 NICVF_RCV_BUF_ALIGN_BYTES, /* alignment */
378 0, /* boundary */
379 DMAP_MAX_PHYSADDR, /* lowaddr */
380 DMAP_MIN_PHYSADDR, /* highaddr */
381 NULL, NULL, /* filtfunc, filtfuncarg */
382 roundup2(buf_size, MCLBYTES), /* maxsize */
383 1, /* nsegments */
384 roundup2(buf_size, MCLBYTES), /* maxsegsize */
385 0, /* flags */
386 NULL, NULL, /* lockfunc, lockfuncarg */
387 &rbdr->rbdr_buff_dmat); /* dmat */
388
389 if (err != 0) {
390 device_printf(nic->dev,
391 "Failed to create busdma tag for RBDR buffers\n");
392 return (err);
393 }
394
395 rbdr->rbdr_buff_dmaps = malloc(sizeof(*rbdr->rbdr_buff_dmaps) *
396 ring_len, M_NICVF, (M_WAITOK | M_ZERO));
397
398 for (idx = 0; idx < ring_len; idx++) {
399 err = bus_dmamap_create(rbdr->rbdr_buff_dmat, 0, &dmap);
400 if (err != 0) {
401 device_printf(nic->dev,
402 "Failed to create DMA map for RB\n");
403 return (err);
404 }
405 rbdr->rbdr_buff_dmaps[idx] = dmap;
406
407 err = nicvf_alloc_rcv_buffer(nic, rbdr, dmap, M_WAITOK,
408 DMA_BUFFER_LEN, &rbuf);
409 if (err != 0)
410 return (err);
411
412 desc = GET_RBDR_DESC(rbdr, idx);
413 desc->buf_addr = (rbuf >> NICVF_RCV_BUF_ALIGN);
414 }
415
416 /* Allocate taskqueue */
417 TASK_INIT(&rbdr->rbdr_task, 0, nicvf_rbdr_task, rbdr);
418 TASK_INIT(&rbdr->rbdr_task_nowait, 0, nicvf_rbdr_task_nowait, rbdr);
419 rbdr->rbdr_taskq = taskqueue_create_fast("nicvf_rbdr_taskq", M_WAITOK,
420 taskqueue_thread_enqueue, &rbdr->rbdr_taskq);
421 taskqueue_start_threads(&rbdr->rbdr_taskq, 1, PI_NET, "%s: rbdr_taskq",
422 device_get_nameunit(nic->dev));
423
424 return (0);
425 }
426
427 /* Free RBDR ring and its receive buffers */
428 static void
nicvf_free_rbdr(struct nicvf * nic,struct rbdr * rbdr)429 nicvf_free_rbdr(struct nicvf *nic, struct rbdr *rbdr)
430 {
431 struct mbuf *mbuf;
432 struct queue_set *qs;
433 struct rbdr_entry_t *desc;
434 struct rbuf_info *rinfo;
435 bus_addr_t buf_addr;
436 int head, tail, idx;
437 int err;
438
439 qs = nic->qs;
440
441 if ((qs == NULL) || (rbdr == NULL))
442 return;
443
444 rbdr->enable = FALSE;
445 if (rbdr->rbdr_taskq != NULL) {
446 /* Remove tasks */
447 while (taskqueue_cancel(rbdr->rbdr_taskq,
448 &rbdr->rbdr_task_nowait, NULL) != 0) {
449 /* Finish the nowait task first */
450 taskqueue_drain(rbdr->rbdr_taskq,
451 &rbdr->rbdr_task_nowait);
452 }
453 taskqueue_free(rbdr->rbdr_taskq);
454 rbdr->rbdr_taskq = NULL;
455
456 while (taskqueue_cancel(taskqueue_thread,
457 &rbdr->rbdr_task, NULL) != 0) {
458 /* Now finish the sleepable task */
459 taskqueue_drain(taskqueue_thread, &rbdr->rbdr_task);
460 }
461 }
462
463 /*
464 * Free all of the memory under the RB descriptors.
465 * There are assumptions here:
466 * 1. Corresponding RBDR is disabled
467 * - it is safe to operate using head and tail indexes
468 * 2. All bffers that were received are properly freed by
469 * the receive handler
470 * - there is no need to unload DMA map and free MBUF for other
471 * descriptors than unused ones
472 */
473 if (rbdr->rbdr_buff_dmat != NULL) {
474 head = rbdr->head;
475 tail = rbdr->tail;
476 while (head != tail) {
477 desc = GET_RBDR_DESC(rbdr, head);
478 buf_addr = desc->buf_addr << NICVF_RCV_BUF_ALIGN;
479 rinfo = GET_RBUF_INFO(PHYS_TO_DMAP(buf_addr));
480 bus_dmamap_unload(rbdr->rbdr_buff_dmat, rinfo->dmap);
481 mbuf = rinfo->mbuf;
482 /* This will destroy everything including rinfo! */
483 m_freem(mbuf);
484 head++;
485 head &= (rbdr->dmem.q_len - 1);
486 }
487 /* Free tail descriptor */
488 desc = GET_RBDR_DESC(rbdr, tail);
489 buf_addr = desc->buf_addr << NICVF_RCV_BUF_ALIGN;
490 rinfo = GET_RBUF_INFO(PHYS_TO_DMAP(buf_addr));
491 bus_dmamap_unload(rbdr->rbdr_buff_dmat, rinfo->dmap);
492 mbuf = rinfo->mbuf;
493 /* This will destroy everything including rinfo! */
494 m_freem(mbuf);
495
496 /* Destroy DMA maps */
497 for (idx = 0; idx < qs->rbdr_len; idx++) {
498 if (rbdr->rbdr_buff_dmaps[idx] == NULL)
499 continue;
500 err = bus_dmamap_destroy(rbdr->rbdr_buff_dmat,
501 rbdr->rbdr_buff_dmaps[idx]);
502 KASSERT(err == 0,
503 ("%s: Could not destroy DMA map for RB, desc: %d",
504 __func__, idx));
505 rbdr->rbdr_buff_dmaps[idx] = NULL;
506 }
507
508 /* Now destroy the tag */
509 err = bus_dma_tag_destroy(rbdr->rbdr_buff_dmat);
510 KASSERT(err == 0,
511 ("%s: Trying to destroy BUSY DMA tag", __func__));
512
513 rbdr->head = 0;
514 rbdr->tail = 0;
515 }
516
517 /* Free RBDR ring */
518 nicvf_free_q_desc_mem(nic, &rbdr->dmem);
519 }
520
521 /*
522 * Refill receive buffer descriptors with new buffers.
523 */
524 static int
nicvf_refill_rbdr(struct rbdr * rbdr,int mflags)525 nicvf_refill_rbdr(struct rbdr *rbdr, int mflags)
526 {
527 struct nicvf *nic;
528 struct queue_set *qs;
529 int rbdr_idx;
530 int tail, qcount;
531 int refill_rb_cnt;
532 struct rbdr_entry_t *desc;
533 bus_dmamap_t dmap;
534 bus_addr_t rbuf;
535 boolean_t rb_alloc_fail;
536 int new_rb;
537
538 rb_alloc_fail = TRUE;
539 new_rb = 0;
540 nic = rbdr->nic;
541 qs = nic->qs;
542 rbdr_idx = rbdr->idx;
543
544 /* Check if it's enabled */
545 if (!rbdr->enable)
546 return (0);
547
548 /* Get no of desc's to be refilled */
549 qcount = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_STATUS0, rbdr_idx);
550 qcount &= 0x7FFFF;
551 /* Doorbell can be ringed with a max of ring size minus 1 */
552 if (qcount >= (qs->rbdr_len - 1)) {
553 rb_alloc_fail = FALSE;
554 goto out;
555 } else
556 refill_rb_cnt = qs->rbdr_len - qcount - 1;
557
558 /* Start filling descs from tail */
559 tail = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_TAIL, rbdr_idx) >> 3;
560 while (refill_rb_cnt) {
561 tail++;
562 tail &= (rbdr->dmem.q_len - 1);
563
564 dmap = rbdr->rbdr_buff_dmaps[tail];
565 if (nicvf_alloc_rcv_buffer(nic, rbdr, dmap, mflags,
566 DMA_BUFFER_LEN, &rbuf)) {
567 /* Something went wrong. Resign */
568 break;
569 }
570 desc = GET_RBDR_DESC(rbdr, tail);
571 desc->buf_addr = (rbuf >> NICVF_RCV_BUF_ALIGN);
572 refill_rb_cnt--;
573 new_rb++;
574 }
575
576 /* make sure all memory stores are done before ringing doorbell */
577 wmb();
578
579 /* Check if buffer allocation failed */
580 if (refill_rb_cnt == 0)
581 rb_alloc_fail = FALSE;
582
583 /* Notify HW */
584 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_DOOR,
585 rbdr_idx, new_rb);
586 out:
587 if (!rb_alloc_fail) {
588 /*
589 * Re-enable RBDR interrupts only
590 * if buffer allocation is success.
591 */
592 nicvf_enable_intr(nic, NICVF_INTR_RBDR, rbdr_idx);
593
594 return (0);
595 }
596
597 return (ENOMEM);
598 }
599
600 /* Refill RBs even if sleep is needed to reclaim memory */
601 static void
nicvf_rbdr_task(void * arg,int pending)602 nicvf_rbdr_task(void *arg, int pending)
603 {
604 struct rbdr *rbdr;
605 int err;
606
607 rbdr = (struct rbdr *)arg;
608
609 err = nicvf_refill_rbdr(rbdr, M_WAITOK);
610 if (__predict_false(err != 0)) {
611 panic("%s: Failed to refill RBs even when sleep enabled",
612 __func__);
613 }
614 }
615
616 /* Refill RBs as soon as possible without waiting */
617 static void
nicvf_rbdr_task_nowait(void * arg,int pending)618 nicvf_rbdr_task_nowait(void *arg, int pending)
619 {
620 struct rbdr *rbdr;
621 int err;
622
623 rbdr = (struct rbdr *)arg;
624
625 err = nicvf_refill_rbdr(rbdr, M_NOWAIT);
626 if (err != 0) {
627 /*
628 * Schedule another, sleepable kernel thread
629 * that will for sure refill the buffers.
630 */
631 taskqueue_enqueue(taskqueue_thread, &rbdr->rbdr_task);
632 }
633 }
634
635 static int
nicvf_rcv_pkt_handler(struct nicvf * nic,struct cmp_queue * cq,struct cqe_rx_t * cqe_rx,int cqe_type)636 nicvf_rcv_pkt_handler(struct nicvf *nic, struct cmp_queue *cq,
637 struct cqe_rx_t *cqe_rx, int cqe_type)
638 {
639 struct mbuf *mbuf;
640 struct rcv_queue *rq;
641 int rq_idx;
642 int err = 0;
643
644 rq_idx = cqe_rx->rq_idx;
645 rq = &nic->qs->rq[rq_idx];
646
647 /* Check for errors */
648 err = nicvf_check_cqe_rx_errs(nic, cq, cqe_rx);
649 if (err && !cqe_rx->rb_cnt)
650 return (0);
651
652 mbuf = nicvf_get_rcv_mbuf(nic, cqe_rx);
653 if (mbuf == NULL) {
654 dprintf(nic->dev, "Packet not received\n");
655 return (0);
656 }
657
658 /* If error packet */
659 if (err != 0) {
660 m_freem(mbuf);
661 return (0);
662 }
663
664 if (rq->lro_enabled &&
665 ((cqe_rx->l3_type == L3TYPE_IPV4) && (cqe_rx->l4_type == L4TYPE_TCP)) &&
666 (mbuf->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
667 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
668 /*
669 * At this point it is known that there are no errors in the
670 * packet. Attempt to LRO enqueue. Send to stack if no resources
671 * or enqueue error.
672 */
673 if ((rq->lro.lro_cnt != 0) &&
674 (tcp_lro_rx(&rq->lro, mbuf, 0) == 0))
675 return (0);
676 }
677 /*
678 * Push this packet to the stack later to avoid
679 * unlocking completion task in the middle of work.
680 */
681 err = buf_ring_enqueue(cq->rx_br, mbuf);
682 if (err != 0) {
683 /*
684 * Failed to enqueue this mbuf.
685 * We don't drop it, just schedule another task.
686 */
687 return (err);
688 }
689
690 return (0);
691 }
692
693 static void
nicvf_snd_pkt_handler(struct nicvf * nic,struct cmp_queue * cq,struct cqe_send_t * cqe_tx,int cqe_type)694 nicvf_snd_pkt_handler(struct nicvf *nic, struct cmp_queue *cq,
695 struct cqe_send_t *cqe_tx, int cqe_type)
696 {
697 bus_dmamap_t dmap;
698 struct mbuf *mbuf;
699 struct snd_queue *sq;
700 struct sq_hdr_subdesc *hdr;
701
702 mbuf = NULL;
703 sq = &nic->qs->sq[cqe_tx->sq_idx];
704
705 hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, cqe_tx->sqe_ptr);
706 if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER)
707 return;
708
709 dprintf(nic->dev,
710 "%s Qset #%d SQ #%d SQ ptr #%d subdesc count %d\n",
711 __func__, cqe_tx->sq_qs, cqe_tx->sq_idx,
712 cqe_tx->sqe_ptr, hdr->subdesc_cnt);
713
714 dmap = (bus_dmamap_t)sq->snd_buff[cqe_tx->sqe_ptr].dmap;
715 bus_dmamap_unload(sq->snd_buff_dmat, dmap);
716
717 mbuf = (struct mbuf *)sq->snd_buff[cqe_tx->sqe_ptr].mbuf;
718 if (mbuf != NULL) {
719 m_freem(mbuf);
720 sq->snd_buff[cqe_tx->sqe_ptr].mbuf = NULL;
721 nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1);
722 }
723
724 nicvf_check_cqe_tx_errs(nic, cq, cqe_tx);
725 }
726
727 static int
nicvf_cq_intr_handler(struct nicvf * nic,uint8_t cq_idx)728 nicvf_cq_intr_handler(struct nicvf *nic, uint8_t cq_idx)
729 {
730 struct mbuf *mbuf;
731 struct ifnet *ifp;
732 int processed_cqe, work_done = 0, tx_done = 0;
733 int cqe_count, cqe_head;
734 struct queue_set *qs = nic->qs;
735 struct cmp_queue *cq = &qs->cq[cq_idx];
736 struct snd_queue *sq = &qs->sq[cq_idx];
737 struct rcv_queue *rq;
738 struct cqe_rx_t *cq_desc;
739 struct lro_ctrl *lro;
740 int rq_idx;
741 int cmp_err;
742
743 NICVF_CMP_LOCK(cq);
744 cmp_err = 0;
745 processed_cqe = 0;
746 /* Get no of valid CQ entries to process */
747 cqe_count = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS, cq_idx);
748 cqe_count &= CQ_CQE_COUNT;
749 if (cqe_count == 0)
750 goto out;
751
752 /* Get head of the valid CQ entries */
753 cqe_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD, cq_idx) >> 9;
754 cqe_head &= 0xFFFF;
755
756 dprintf(nic->dev, "%s CQ%d cqe_count %d cqe_head %d\n",
757 __func__, cq_idx, cqe_count, cqe_head);
758 while (processed_cqe < cqe_count) {
759 /* Get the CQ descriptor */
760 cq_desc = (struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head);
761 cqe_head++;
762 cqe_head &= (cq->dmem.q_len - 1);
763 /* Prefetch next CQ descriptor */
764 __builtin_prefetch((struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head));
765
766 dprintf(nic->dev, "CQ%d cq_desc->cqe_type %d\n", cq_idx,
767 cq_desc->cqe_type);
768 switch (cq_desc->cqe_type) {
769 case CQE_TYPE_RX:
770 cmp_err = nicvf_rcv_pkt_handler(nic, cq, cq_desc,
771 CQE_TYPE_RX);
772 if (__predict_false(cmp_err != 0)) {
773 /*
774 * Ups. Cannot finish now.
775 * Let's try again later.
776 */
777 goto done;
778 }
779 work_done++;
780 break;
781 case CQE_TYPE_SEND:
782 nicvf_snd_pkt_handler(nic, cq, (void *)cq_desc,
783 CQE_TYPE_SEND);
784 tx_done++;
785 break;
786 case CQE_TYPE_INVALID:
787 case CQE_TYPE_RX_SPLIT:
788 case CQE_TYPE_RX_TCP:
789 case CQE_TYPE_SEND_PTP:
790 /* Ignore for now */
791 break;
792 }
793 processed_cqe++;
794 }
795 done:
796 dprintf(nic->dev,
797 "%s CQ%d processed_cqe %d work_done %d\n",
798 __func__, cq_idx, processed_cqe, work_done);
799
800 /* Ring doorbell to inform H/W to reuse processed CQEs */
801 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_DOOR, cq_idx, processed_cqe);
802
803 if ((tx_done > 0) &&
804 ((if_getdrvflags(nic->ifp) & IFF_DRV_RUNNING) != 0)) {
805 /* Reenable TXQ if its stopped earlier due to SQ full */
806 if_setdrvflagbits(nic->ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
807 taskqueue_enqueue(sq->snd_taskq, &sq->snd_task);
808 }
809 out:
810 /*
811 * Flush any outstanding LRO work
812 */
813 rq_idx = cq_idx;
814 rq = &nic->qs->rq[rq_idx];
815 lro = &rq->lro;
816 tcp_lro_flush_all(lro);
817
818 NICVF_CMP_UNLOCK(cq);
819
820 ifp = nic->ifp;
821 /* Push received MBUFs to the stack */
822 while (!buf_ring_empty(cq->rx_br)) {
823 mbuf = buf_ring_dequeue_mc(cq->rx_br);
824 if (__predict_true(mbuf != NULL))
825 (*ifp->if_input)(ifp, mbuf);
826 }
827
828 return (cmp_err);
829 }
830
831 /*
832 * Qset error interrupt handler
833 *
834 * As of now only CQ errors are handled
835 */
836 static void
nicvf_qs_err_task(void * arg,int pending)837 nicvf_qs_err_task(void *arg, int pending)
838 {
839 struct nicvf *nic;
840 struct queue_set *qs;
841 int qidx;
842 uint64_t status;
843 boolean_t enable = TRUE;
844
845 nic = (struct nicvf *)arg;
846 qs = nic->qs;
847
848 /* Deactivate network interface */
849 if_setdrvflagbits(nic->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
850
851 /* Check if it is CQ err */
852 for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
853 status = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS,
854 qidx);
855 if ((status & CQ_ERR_MASK) == 0)
856 continue;
857 /* Process already queued CQEs and reconfig CQ */
858 nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
859 nicvf_sq_disable(nic, qidx);
860 (void)nicvf_cq_intr_handler(nic, qidx);
861 nicvf_cmp_queue_config(nic, qs, qidx, enable);
862 nicvf_sq_free_used_descs(nic, &qs->sq[qidx], qidx);
863 nicvf_sq_enable(nic, &qs->sq[qidx], qidx);
864 nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx);
865 }
866
867 if_setdrvflagbits(nic->ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
868 /* Re-enable Qset error interrupt */
869 nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0);
870 }
871
872 static void
nicvf_cmp_task(void * arg,int pending)873 nicvf_cmp_task(void *arg, int pending)
874 {
875 struct cmp_queue *cq;
876 struct nicvf *nic;
877 int cmp_err;
878
879 cq = (struct cmp_queue *)arg;
880 nic = cq->nic;
881
882 /* Handle CQ descriptors */
883 cmp_err = nicvf_cq_intr_handler(nic, cq->idx);
884 if (__predict_false(cmp_err != 0)) {
885 /*
886 * Schedule another thread here since we did not
887 * process the entire CQ due to Tx or Rx CQ parse error.
888 */
889 taskqueue_enqueue(cq->cmp_taskq, &cq->cmp_task);
890 }
891
892 nicvf_clear_intr(nic, NICVF_INTR_CQ, cq->idx);
893 /* Reenable interrupt (previously disabled in nicvf_intr_handler() */
894 nicvf_enable_intr(nic, NICVF_INTR_CQ, cq->idx);
895
896 }
897
898 /* Initialize completion queue */
899 static int
nicvf_init_cmp_queue(struct nicvf * nic,struct cmp_queue * cq,int q_len,int qidx)900 nicvf_init_cmp_queue(struct nicvf *nic, struct cmp_queue *cq, int q_len,
901 int qidx)
902 {
903 int err;
904
905 /* Initizalize lock */
906 snprintf(cq->mtx_name, sizeof(cq->mtx_name), "%s: CQ(%d) lock",
907 device_get_nameunit(nic->dev), qidx);
908 mtx_init(&cq->mtx, cq->mtx_name, NULL, MTX_DEF);
909
910 err = nicvf_alloc_q_desc_mem(nic, &cq->dmem, q_len, CMP_QUEUE_DESC_SIZE,
911 NICVF_CQ_BASE_ALIGN_BYTES);
912
913 if (err != 0) {
914 device_printf(nic->dev,
915 "Could not allocate DMA memory for CQ\n");
916 return (err);
917 }
918
919 cq->desc = cq->dmem.base;
920 cq->thresh = pass1_silicon(nic->dev) ? 0 : CMP_QUEUE_CQE_THRESH;
921 cq->nic = nic;
922 cq->idx = qidx;
923 nic->cq_coalesce_usecs = (CMP_QUEUE_TIMER_THRESH * 0.05) - 1;
924
925 cq->rx_br = buf_ring_alloc(CMP_QUEUE_LEN * 8, M_DEVBUF, M_WAITOK,
926 &cq->mtx);
927
928 /* Allocate taskqueue */
929 NET_TASK_INIT(&cq->cmp_task, 0, nicvf_cmp_task, cq);
930 cq->cmp_taskq = taskqueue_create_fast("nicvf_cmp_taskq", M_WAITOK,
931 taskqueue_thread_enqueue, &cq->cmp_taskq);
932 taskqueue_start_threads(&cq->cmp_taskq, 1, PI_NET, "%s: cmp_taskq(%d)",
933 device_get_nameunit(nic->dev), qidx);
934
935 return (0);
936 }
937
938 static void
nicvf_free_cmp_queue(struct nicvf * nic,struct cmp_queue * cq)939 nicvf_free_cmp_queue(struct nicvf *nic, struct cmp_queue *cq)
940 {
941
942 if (cq == NULL)
943 return;
944 /*
945 * The completion queue itself should be disabled by now
946 * (ref. nicvf_snd_queue_config()).
947 * Ensure that it is safe to disable it or panic.
948 */
949 if (cq->enable)
950 panic("%s: Trying to free working CQ(%d)", __func__, cq->idx);
951
952 if (cq->cmp_taskq != NULL) {
953 /* Remove task */
954 while (taskqueue_cancel(cq->cmp_taskq, &cq->cmp_task, NULL) != 0)
955 taskqueue_drain(cq->cmp_taskq, &cq->cmp_task);
956
957 taskqueue_free(cq->cmp_taskq);
958 cq->cmp_taskq = NULL;
959 }
960 /*
961 * Completion interrupt will possibly enable interrupts again
962 * so disable interrupting now after we finished processing
963 * completion task. It is safe to do so since the corresponding CQ
964 * was already disabled.
965 */
966 nicvf_disable_intr(nic, NICVF_INTR_CQ, cq->idx);
967 nicvf_clear_intr(nic, NICVF_INTR_CQ, cq->idx);
968
969 NICVF_CMP_LOCK(cq);
970 nicvf_free_q_desc_mem(nic, &cq->dmem);
971 drbr_free(cq->rx_br, M_DEVBUF);
972 NICVF_CMP_UNLOCK(cq);
973 mtx_destroy(&cq->mtx);
974 memset(cq->mtx_name, 0, sizeof(cq->mtx_name));
975 }
976
977 int
nicvf_xmit_locked(struct snd_queue * sq)978 nicvf_xmit_locked(struct snd_queue *sq)
979 {
980 struct nicvf *nic;
981 struct ifnet *ifp;
982 struct mbuf *next;
983 int err;
984
985 NICVF_TX_LOCK_ASSERT(sq);
986
987 nic = sq->nic;
988 ifp = nic->ifp;
989 err = 0;
990
991 while ((next = drbr_peek(ifp, sq->br)) != NULL) {
992 /* Send a copy of the frame to the BPF listener */
993 ETHER_BPF_MTAP(ifp, next);
994
995 err = nicvf_tx_mbuf_locked(sq, &next);
996 if (err != 0) {
997 if (next == NULL)
998 drbr_advance(ifp, sq->br);
999 else
1000 drbr_putback(ifp, sq->br, next);
1001
1002 break;
1003 }
1004 drbr_advance(ifp, sq->br);
1005 }
1006 return (err);
1007 }
1008
1009 static void
nicvf_snd_task(void * arg,int pending)1010 nicvf_snd_task(void *arg, int pending)
1011 {
1012 struct snd_queue *sq = (struct snd_queue *)arg;
1013 struct nicvf *nic;
1014 struct ifnet *ifp;
1015 int err;
1016
1017 nic = sq->nic;
1018 ifp = nic->ifp;
1019
1020 /*
1021 * Skip sending anything if the driver is not running,
1022 * SQ full or link is down.
1023 */
1024 if (((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
1025 IFF_DRV_RUNNING) || !nic->link_up)
1026 return;
1027
1028 NICVF_TX_LOCK(sq);
1029 err = nicvf_xmit_locked(sq);
1030 NICVF_TX_UNLOCK(sq);
1031 /* Try again */
1032 if (err != 0)
1033 taskqueue_enqueue(sq->snd_taskq, &sq->snd_task);
1034 }
1035
1036 /* Initialize transmit queue */
1037 static int
nicvf_init_snd_queue(struct nicvf * nic,struct snd_queue * sq,int q_len,int qidx)1038 nicvf_init_snd_queue(struct nicvf *nic, struct snd_queue *sq, int q_len,
1039 int qidx)
1040 {
1041 size_t i;
1042 int err;
1043
1044 /* Initizalize TX lock for this queue */
1045 snprintf(sq->mtx_name, sizeof(sq->mtx_name), "%s: SQ(%d) lock",
1046 device_get_nameunit(nic->dev), qidx);
1047 mtx_init(&sq->mtx, sq->mtx_name, NULL, MTX_DEF);
1048
1049 NICVF_TX_LOCK(sq);
1050 /* Allocate buffer ring */
1051 sq->br = buf_ring_alloc(q_len / MIN_SQ_DESC_PER_PKT_XMIT, M_DEVBUF,
1052 M_NOWAIT, &sq->mtx);
1053 if (sq->br == NULL) {
1054 device_printf(nic->dev,
1055 "ERROR: Could not set up buf ring for SQ(%d)\n", qidx);
1056 err = ENOMEM;
1057 goto error;
1058 }
1059
1060 /* Allocate DMA memory for Tx descriptors */
1061 err = nicvf_alloc_q_desc_mem(nic, &sq->dmem, q_len, SND_QUEUE_DESC_SIZE,
1062 NICVF_SQ_BASE_ALIGN_BYTES);
1063 if (err != 0) {
1064 device_printf(nic->dev,
1065 "Could not allocate DMA memory for SQ\n");
1066 goto error;
1067 }
1068
1069 sq->desc = sq->dmem.base;
1070 sq->head = sq->tail = 0;
1071 atomic_store_rel_int(&sq->free_cnt, q_len - 1);
1072 sq->thresh = SND_QUEUE_THRESH;
1073 sq->idx = qidx;
1074 sq->nic = nic;
1075
1076 /*
1077 * Allocate DMA maps for Tx buffers
1078 */
1079
1080 /* Create DMA tag first */
1081 err = bus_dma_tag_create(
1082 bus_get_dma_tag(nic->dev), /* parent tag */
1083 1, /* alignment */
1084 0, /* boundary */
1085 BUS_SPACE_MAXADDR, /* lowaddr */
1086 BUS_SPACE_MAXADDR, /* highaddr */
1087 NULL, NULL, /* filtfunc, filtfuncarg */
1088 NICVF_TSO_MAXSIZE, /* maxsize */
1089 NICVF_TSO_NSEGS, /* nsegments */
1090 MCLBYTES, /* maxsegsize */
1091 0, /* flags */
1092 NULL, NULL, /* lockfunc, lockfuncarg */
1093 &sq->snd_buff_dmat); /* dmat */
1094
1095 if (err != 0) {
1096 device_printf(nic->dev,
1097 "Failed to create busdma tag for Tx buffers\n");
1098 goto error;
1099 }
1100
1101 /* Allocate send buffers array */
1102 sq->snd_buff = malloc(sizeof(*sq->snd_buff) * q_len, M_NICVF,
1103 (M_NOWAIT | M_ZERO));
1104 if (sq->snd_buff == NULL) {
1105 device_printf(nic->dev,
1106 "Could not allocate memory for Tx buffers array\n");
1107 err = ENOMEM;
1108 goto error;
1109 }
1110
1111 /* Now populate maps */
1112 for (i = 0; i < q_len; i++) {
1113 err = bus_dmamap_create(sq->snd_buff_dmat, 0,
1114 &sq->snd_buff[i].dmap);
1115 if (err != 0) {
1116 device_printf(nic->dev,
1117 "Failed to create DMA maps for Tx buffers\n");
1118 goto error;
1119 }
1120 }
1121 NICVF_TX_UNLOCK(sq);
1122
1123 /* Allocate taskqueue */
1124 TASK_INIT(&sq->snd_task, 0, nicvf_snd_task, sq);
1125 sq->snd_taskq = taskqueue_create_fast("nicvf_snd_taskq", M_WAITOK,
1126 taskqueue_thread_enqueue, &sq->snd_taskq);
1127 taskqueue_start_threads(&sq->snd_taskq, 1, PI_NET, "%s: snd_taskq(%d)",
1128 device_get_nameunit(nic->dev), qidx);
1129
1130 return (0);
1131 error:
1132 NICVF_TX_UNLOCK(sq);
1133 return (err);
1134 }
1135
1136 static void
nicvf_free_snd_queue(struct nicvf * nic,struct snd_queue * sq)1137 nicvf_free_snd_queue(struct nicvf *nic, struct snd_queue *sq)
1138 {
1139 struct queue_set *qs = nic->qs;
1140 size_t i;
1141 int err;
1142
1143 if (sq == NULL)
1144 return;
1145
1146 if (sq->snd_taskq != NULL) {
1147 /* Remove task */
1148 while (taskqueue_cancel(sq->snd_taskq, &sq->snd_task, NULL) != 0)
1149 taskqueue_drain(sq->snd_taskq, &sq->snd_task);
1150
1151 taskqueue_free(sq->snd_taskq);
1152 sq->snd_taskq = NULL;
1153 }
1154
1155 NICVF_TX_LOCK(sq);
1156 if (sq->snd_buff_dmat != NULL) {
1157 if (sq->snd_buff != NULL) {
1158 for (i = 0; i < qs->sq_len; i++) {
1159 m_freem(sq->snd_buff[i].mbuf);
1160 sq->snd_buff[i].mbuf = NULL;
1161
1162 bus_dmamap_unload(sq->snd_buff_dmat,
1163 sq->snd_buff[i].dmap);
1164 err = bus_dmamap_destroy(sq->snd_buff_dmat,
1165 sq->snd_buff[i].dmap);
1166 /*
1167 * If bus_dmamap_destroy fails it can cause
1168 * random panic later if the tag is also
1169 * destroyed in the process.
1170 */
1171 KASSERT(err == 0,
1172 ("%s: Could not destroy DMA map for SQ",
1173 __func__));
1174 }
1175 }
1176
1177 free(sq->snd_buff, M_NICVF);
1178
1179 err = bus_dma_tag_destroy(sq->snd_buff_dmat);
1180 KASSERT(err == 0,
1181 ("%s: Trying to destroy BUSY DMA tag", __func__));
1182 }
1183
1184 /* Free private driver ring for this send queue */
1185 if (sq->br != NULL)
1186 drbr_free(sq->br, M_DEVBUF);
1187
1188 if (sq->dmem.base != NULL)
1189 nicvf_free_q_desc_mem(nic, &sq->dmem);
1190
1191 NICVF_TX_UNLOCK(sq);
1192 /* Destroy Tx lock */
1193 mtx_destroy(&sq->mtx);
1194 memset(sq->mtx_name, 0, sizeof(sq->mtx_name));
1195 }
1196
1197 static void
nicvf_reclaim_snd_queue(struct nicvf * nic,struct queue_set * qs,int qidx)1198 nicvf_reclaim_snd_queue(struct nicvf *nic, struct queue_set *qs, int qidx)
1199 {
1200
1201 /* Disable send queue */
1202 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, 0);
1203 /* Check if SQ is stopped */
1204 if (nicvf_poll_reg(nic, qidx, NIC_QSET_SQ_0_7_STATUS, 21, 1, 0x01))
1205 return;
1206 /* Reset send queue */
1207 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, NICVF_SQ_RESET);
1208 }
1209
1210 static void
nicvf_reclaim_rcv_queue(struct nicvf * nic,struct queue_set * qs,int qidx)1211 nicvf_reclaim_rcv_queue(struct nicvf *nic, struct queue_set *qs, int qidx)
1212 {
1213 union nic_mbx mbx = {};
1214
1215 /* Make sure all packets in the pipeline are written back into mem */
1216 mbx.msg.msg = NIC_MBOX_MSG_RQ_SW_SYNC;
1217 nicvf_send_msg_to_pf(nic, &mbx);
1218 }
1219
1220 static void
nicvf_reclaim_cmp_queue(struct nicvf * nic,struct queue_set * qs,int qidx)1221 nicvf_reclaim_cmp_queue(struct nicvf *nic, struct queue_set *qs, int qidx)
1222 {
1223
1224 /* Disable timer threshold (doesn't get reset upon CQ reset */
1225 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG2, qidx, 0);
1226 /* Disable completion queue */
1227 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, 0);
1228 /* Reset completion queue */
1229 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, NICVF_CQ_RESET);
1230 }
1231
1232 static void
nicvf_reclaim_rbdr(struct nicvf * nic,struct rbdr * rbdr,int qidx)1233 nicvf_reclaim_rbdr(struct nicvf *nic, struct rbdr *rbdr, int qidx)
1234 {
1235 uint64_t tmp, fifo_state;
1236 int timeout = 10;
1237
1238 /* Save head and tail pointers for feeing up buffers */
1239 rbdr->head =
1240 nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_HEAD, qidx) >> 3;
1241 rbdr->tail =
1242 nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_TAIL, qidx) >> 3;
1243
1244 /*
1245 * If RBDR FIFO is in 'FAIL' state then do a reset first
1246 * before relaiming.
1247 */
1248 fifo_state = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_STATUS0, qidx);
1249 if (((fifo_state >> 62) & 0x03) == 0x3) {
1250 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG,
1251 qidx, NICVF_RBDR_RESET);
1252 }
1253
1254 /* Disable RBDR */
1255 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx, 0);
1256 if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x00))
1257 return;
1258 while (1) {
1259 tmp = nicvf_queue_reg_read(nic,
1260 NIC_QSET_RBDR_0_1_PREFETCH_STATUS, qidx);
1261 if ((tmp & 0xFFFFFFFF) == ((tmp >> 32) & 0xFFFFFFFF))
1262 break;
1263
1264 DELAY(1000);
1265 timeout--;
1266 if (!timeout) {
1267 device_printf(nic->dev,
1268 "Failed polling on prefetch status\n");
1269 return;
1270 }
1271 }
1272 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx,
1273 NICVF_RBDR_RESET);
1274
1275 if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x02))
1276 return;
1277 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx, 0x00);
1278 if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x00))
1279 return;
1280 }
1281
1282 /* Configures receive queue */
1283 static void
nicvf_rcv_queue_config(struct nicvf * nic,struct queue_set * qs,int qidx,bool enable)1284 nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs,
1285 int qidx, bool enable)
1286 {
1287 union nic_mbx mbx = {};
1288 struct rcv_queue *rq;
1289 struct rq_cfg rq_cfg;
1290 struct ifnet *ifp;
1291 struct lro_ctrl *lro;
1292
1293 ifp = nic->ifp;
1294
1295 rq = &qs->rq[qidx];
1296 rq->enable = enable;
1297
1298 lro = &rq->lro;
1299
1300 /* Disable receive queue */
1301 nicvf_queue_reg_write(nic, NIC_QSET_RQ_0_7_CFG, qidx, 0);
1302
1303 if (!rq->enable) {
1304 nicvf_reclaim_rcv_queue(nic, qs, qidx);
1305 /* Free LRO memory */
1306 tcp_lro_free(lro);
1307 rq->lro_enabled = FALSE;
1308 return;
1309 }
1310
1311 /* Configure LRO if enabled */
1312 rq->lro_enabled = FALSE;
1313 if ((if_getcapenable(ifp) & IFCAP_LRO) != 0) {
1314 if (tcp_lro_init(lro) != 0) {
1315 device_printf(nic->dev,
1316 "Failed to initialize LRO for RXQ%d\n", qidx);
1317 } else {
1318 rq->lro_enabled = TRUE;
1319 lro->ifp = nic->ifp;
1320 }
1321 }
1322
1323 rq->cq_qs = qs->vnic_id;
1324 rq->cq_idx = qidx;
1325 rq->start_rbdr_qs = qs->vnic_id;
1326 rq->start_qs_rbdr_idx = qs->rbdr_cnt - 1;
1327 rq->cont_rbdr_qs = qs->vnic_id;
1328 rq->cont_qs_rbdr_idx = qs->rbdr_cnt - 1;
1329 /* all writes of RBDR data to be loaded into L2 Cache as well*/
1330 rq->caching = 1;
1331
1332 /* Send a mailbox msg to PF to config RQ */
1333 mbx.rq.msg = NIC_MBOX_MSG_RQ_CFG;
1334 mbx.rq.qs_num = qs->vnic_id;
1335 mbx.rq.rq_num = qidx;
1336 mbx.rq.cfg = ((uint64_t)rq->caching << 26) | (rq->cq_qs << 19) |
1337 (rq->cq_idx << 16) | (rq->cont_rbdr_qs << 9) |
1338 (rq->cont_qs_rbdr_idx << 8) | (rq->start_rbdr_qs << 1) |
1339 (rq->start_qs_rbdr_idx);
1340 nicvf_send_msg_to_pf(nic, &mbx);
1341
1342 mbx.rq.msg = NIC_MBOX_MSG_RQ_BP_CFG;
1343 mbx.rq.cfg = (1UL << 63) | (1UL << 62) | (qs->vnic_id << 0);
1344 nicvf_send_msg_to_pf(nic, &mbx);
1345
1346 /*
1347 * RQ drop config
1348 * Enable CQ drop to reserve sufficient CQEs for all tx packets
1349 */
1350 mbx.rq.msg = NIC_MBOX_MSG_RQ_DROP_CFG;
1351 mbx.rq.cfg = (1UL << 62) | (RQ_CQ_DROP << 8);
1352 nicvf_send_msg_to_pf(nic, &mbx);
1353
1354 nicvf_queue_reg_write(nic, NIC_QSET_RQ_GEN_CFG, 0, 0x00);
1355
1356 /* Enable Receive queue */
1357 rq_cfg.ena = 1;
1358 rq_cfg.tcp_ena = 0;
1359 nicvf_queue_reg_write(nic, NIC_QSET_RQ_0_7_CFG, qidx,
1360 *(uint64_t *)&rq_cfg);
1361 }
1362
1363 /* Configures completion queue */
1364 static void
nicvf_cmp_queue_config(struct nicvf * nic,struct queue_set * qs,int qidx,boolean_t enable)1365 nicvf_cmp_queue_config(struct nicvf *nic, struct queue_set *qs,
1366 int qidx, boolean_t enable)
1367 {
1368 struct cmp_queue *cq;
1369 struct cq_cfg cq_cfg;
1370
1371 cq = &qs->cq[qidx];
1372 cq->enable = enable;
1373
1374 if (!cq->enable) {
1375 nicvf_reclaim_cmp_queue(nic, qs, qidx);
1376 return;
1377 }
1378
1379 /* Reset completion queue */
1380 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, NICVF_CQ_RESET);
1381
1382 /* Set completion queue base address */
1383 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_BASE, qidx,
1384 (uint64_t)(cq->dmem.phys_base));
1385
1386 /* Enable Completion queue */
1387 cq_cfg.ena = 1;
1388 cq_cfg.reset = 0;
1389 cq_cfg.caching = 0;
1390 cq_cfg.qsize = CMP_QSIZE;
1391 cq_cfg.avg_con = 0;
1392 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, *(uint64_t *)&cq_cfg);
1393
1394 /* Set threshold value for interrupt generation */
1395 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_THRESH, qidx, cq->thresh);
1396 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG2, qidx,
1397 nic->cq_coalesce_usecs);
1398 }
1399
1400 /* Configures transmit queue */
1401 static void
nicvf_snd_queue_config(struct nicvf * nic,struct queue_set * qs,int qidx,boolean_t enable)1402 nicvf_snd_queue_config(struct nicvf *nic, struct queue_set *qs, int qidx,
1403 boolean_t enable)
1404 {
1405 union nic_mbx mbx = {};
1406 struct snd_queue *sq;
1407 struct sq_cfg sq_cfg;
1408
1409 sq = &qs->sq[qidx];
1410 sq->enable = enable;
1411
1412 if (!sq->enable) {
1413 nicvf_reclaim_snd_queue(nic, qs, qidx);
1414 return;
1415 }
1416
1417 /* Reset send queue */
1418 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, NICVF_SQ_RESET);
1419
1420 sq->cq_qs = qs->vnic_id;
1421 sq->cq_idx = qidx;
1422
1423 /* Send a mailbox msg to PF to config SQ */
1424 mbx.sq.msg = NIC_MBOX_MSG_SQ_CFG;
1425 mbx.sq.qs_num = qs->vnic_id;
1426 mbx.sq.sq_num = qidx;
1427 mbx.sq.sqs_mode = nic->sqs_mode;
1428 mbx.sq.cfg = (sq->cq_qs << 3) | sq->cq_idx;
1429 nicvf_send_msg_to_pf(nic, &mbx);
1430
1431 /* Set queue base address */
1432 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_BASE, qidx,
1433 (uint64_t)(sq->dmem.phys_base));
1434
1435 /* Enable send queue & set queue size */
1436 sq_cfg.ena = 1;
1437 sq_cfg.reset = 0;
1438 sq_cfg.ldwb = 0;
1439 sq_cfg.qsize = SND_QSIZE;
1440 sq_cfg.tstmp_bgx_intf = 0;
1441 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, *(uint64_t *)&sq_cfg);
1442
1443 /* Set threshold value for interrupt generation */
1444 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_THRESH, qidx, sq->thresh);
1445 }
1446
1447 /* Configures receive buffer descriptor ring */
1448 static void
nicvf_rbdr_config(struct nicvf * nic,struct queue_set * qs,int qidx,boolean_t enable)1449 nicvf_rbdr_config(struct nicvf *nic, struct queue_set *qs, int qidx,
1450 boolean_t enable)
1451 {
1452 struct rbdr *rbdr;
1453 struct rbdr_cfg rbdr_cfg;
1454
1455 rbdr = &qs->rbdr[qidx];
1456 nicvf_reclaim_rbdr(nic, rbdr, qidx);
1457 if (!enable)
1458 return;
1459
1460 /* Set descriptor base address */
1461 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_BASE, qidx,
1462 (uint64_t)(rbdr->dmem.phys_base));
1463
1464 /* Enable RBDR & set queue size */
1465 /* Buffer size should be in multiples of 128 bytes */
1466 rbdr_cfg.ena = 1;
1467 rbdr_cfg.reset = 0;
1468 rbdr_cfg.ldwb = 0;
1469 rbdr_cfg.qsize = RBDR_SIZE;
1470 rbdr_cfg.avg_con = 0;
1471 rbdr_cfg.lines = rbdr->dma_size / 128;
1472 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx,
1473 *(uint64_t *)&rbdr_cfg);
1474
1475 /* Notify HW */
1476 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_DOOR, qidx,
1477 qs->rbdr_len - 1);
1478
1479 /* Set threshold value for interrupt generation */
1480 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_THRESH, qidx,
1481 rbdr->thresh - 1);
1482 }
1483
1484 /* Requests PF to assign and enable Qset */
1485 void
nicvf_qset_config(struct nicvf * nic,boolean_t enable)1486 nicvf_qset_config(struct nicvf *nic, boolean_t enable)
1487 {
1488 union nic_mbx mbx = {};
1489 struct queue_set *qs;
1490 struct qs_cfg *qs_cfg;
1491
1492 qs = nic->qs;
1493 if (qs == NULL) {
1494 device_printf(nic->dev,
1495 "Qset is still not allocated, don't init queues\n");
1496 return;
1497 }
1498
1499 qs->enable = enable;
1500 qs->vnic_id = nic->vf_id;
1501
1502 /* Send a mailbox msg to PF to config Qset */
1503 mbx.qs.msg = NIC_MBOX_MSG_QS_CFG;
1504 mbx.qs.num = qs->vnic_id;
1505
1506 mbx.qs.cfg = 0;
1507 qs_cfg = (struct qs_cfg *)&mbx.qs.cfg;
1508 if (qs->enable) {
1509 qs_cfg->ena = 1;
1510 qs_cfg->vnic = qs->vnic_id;
1511 }
1512 nicvf_send_msg_to_pf(nic, &mbx);
1513 }
1514
1515 static void
nicvf_free_resources(struct nicvf * nic)1516 nicvf_free_resources(struct nicvf *nic)
1517 {
1518 int qidx;
1519 struct queue_set *qs;
1520
1521 qs = nic->qs;
1522 /*
1523 * Remove QS error task first since it has to be dead
1524 * to safely free completion queue tasks.
1525 */
1526 if (qs->qs_err_taskq != NULL) {
1527 /* Shut down QS error tasks */
1528 while (taskqueue_cancel(qs->qs_err_taskq,
1529 &qs->qs_err_task, NULL) != 0) {
1530 taskqueue_drain(qs->qs_err_taskq, &qs->qs_err_task);
1531 }
1532 taskqueue_free(qs->qs_err_taskq);
1533 qs->qs_err_taskq = NULL;
1534 }
1535 /* Free receive buffer descriptor ring */
1536 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++)
1537 nicvf_free_rbdr(nic, &qs->rbdr[qidx]);
1538
1539 /* Free completion queue */
1540 for (qidx = 0; qidx < qs->cq_cnt; qidx++)
1541 nicvf_free_cmp_queue(nic, &qs->cq[qidx]);
1542
1543 /* Free send queue */
1544 for (qidx = 0; qidx < qs->sq_cnt; qidx++)
1545 nicvf_free_snd_queue(nic, &qs->sq[qidx]);
1546 }
1547
1548 static int
nicvf_alloc_resources(struct nicvf * nic)1549 nicvf_alloc_resources(struct nicvf *nic)
1550 {
1551 struct queue_set *qs = nic->qs;
1552 int qidx;
1553
1554 /* Alloc receive buffer descriptor ring */
1555 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) {
1556 if (nicvf_init_rbdr(nic, &qs->rbdr[qidx], qs->rbdr_len,
1557 DMA_BUFFER_LEN, qidx))
1558 goto alloc_fail;
1559 }
1560
1561 /* Alloc send queue */
1562 for (qidx = 0; qidx < qs->sq_cnt; qidx++) {
1563 if (nicvf_init_snd_queue(nic, &qs->sq[qidx], qs->sq_len, qidx))
1564 goto alloc_fail;
1565 }
1566
1567 /* Alloc completion queue */
1568 for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
1569 if (nicvf_init_cmp_queue(nic, &qs->cq[qidx], qs->cq_len, qidx))
1570 goto alloc_fail;
1571 }
1572
1573 /* Allocate QS error taskqueue */
1574 NET_TASK_INIT(&qs->qs_err_task, 0, nicvf_qs_err_task, nic);
1575 qs->qs_err_taskq = taskqueue_create_fast("nicvf_qs_err_taskq", M_WAITOK,
1576 taskqueue_thread_enqueue, &qs->qs_err_taskq);
1577 taskqueue_start_threads(&qs->qs_err_taskq, 1, PI_NET, "%s: qs_taskq",
1578 device_get_nameunit(nic->dev));
1579
1580 return (0);
1581 alloc_fail:
1582 nicvf_free_resources(nic);
1583 return (ENOMEM);
1584 }
1585
1586 int
nicvf_set_qset_resources(struct nicvf * nic)1587 nicvf_set_qset_resources(struct nicvf *nic)
1588 {
1589 struct queue_set *qs;
1590
1591 qs = malloc(sizeof(*qs), M_NICVF, (M_ZERO | M_WAITOK));
1592 nic->qs = qs;
1593
1594 /* Set count of each queue */
1595 qs->rbdr_cnt = RBDR_CNT;
1596 qs->rq_cnt = RCV_QUEUE_CNT;
1597
1598 qs->sq_cnt = SND_QUEUE_CNT;
1599 qs->cq_cnt = CMP_QUEUE_CNT;
1600
1601 /* Set queue lengths */
1602 qs->rbdr_len = RCV_BUF_COUNT;
1603 qs->sq_len = SND_QUEUE_LEN;
1604 qs->cq_len = CMP_QUEUE_LEN;
1605
1606 nic->rx_queues = qs->rq_cnt;
1607 nic->tx_queues = qs->sq_cnt;
1608
1609 return (0);
1610 }
1611
1612 int
nicvf_config_data_transfer(struct nicvf * nic,boolean_t enable)1613 nicvf_config_data_transfer(struct nicvf *nic, boolean_t enable)
1614 {
1615 boolean_t disable = FALSE;
1616 struct queue_set *qs;
1617 int qidx;
1618
1619 qs = nic->qs;
1620 if (qs == NULL)
1621 return (0);
1622
1623 if (enable) {
1624 if (nicvf_alloc_resources(nic) != 0)
1625 return (ENOMEM);
1626
1627 for (qidx = 0; qidx < qs->sq_cnt; qidx++)
1628 nicvf_snd_queue_config(nic, qs, qidx, enable);
1629 for (qidx = 0; qidx < qs->cq_cnt; qidx++)
1630 nicvf_cmp_queue_config(nic, qs, qidx, enable);
1631 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++)
1632 nicvf_rbdr_config(nic, qs, qidx, enable);
1633 for (qidx = 0; qidx < qs->rq_cnt; qidx++)
1634 nicvf_rcv_queue_config(nic, qs, qidx, enable);
1635 } else {
1636 for (qidx = 0; qidx < qs->rq_cnt; qidx++)
1637 nicvf_rcv_queue_config(nic, qs, qidx, disable);
1638 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++)
1639 nicvf_rbdr_config(nic, qs, qidx, disable);
1640 for (qidx = 0; qidx < qs->sq_cnt; qidx++)
1641 nicvf_snd_queue_config(nic, qs, qidx, disable);
1642 for (qidx = 0; qidx < qs->cq_cnt; qidx++)
1643 nicvf_cmp_queue_config(nic, qs, qidx, disable);
1644
1645 nicvf_free_resources(nic);
1646 }
1647
1648 return (0);
1649 }
1650
1651 /*
1652 * Get a free desc from SQ
1653 * returns descriptor ponter & descriptor number
1654 */
1655 static __inline int
nicvf_get_sq_desc(struct snd_queue * sq,int desc_cnt)1656 nicvf_get_sq_desc(struct snd_queue *sq, int desc_cnt)
1657 {
1658 int qentry;
1659
1660 qentry = sq->tail;
1661 atomic_subtract_int(&sq->free_cnt, desc_cnt);
1662 sq->tail += desc_cnt;
1663 sq->tail &= (sq->dmem.q_len - 1);
1664
1665 return (qentry);
1666 }
1667
1668 /* Free descriptor back to SQ for future use */
1669 static void
nicvf_put_sq_desc(struct snd_queue * sq,int desc_cnt)1670 nicvf_put_sq_desc(struct snd_queue *sq, int desc_cnt)
1671 {
1672
1673 atomic_add_int(&sq->free_cnt, desc_cnt);
1674 sq->head += desc_cnt;
1675 sq->head &= (sq->dmem.q_len - 1);
1676 }
1677
1678 static __inline int
nicvf_get_nxt_sqentry(struct snd_queue * sq,int qentry)1679 nicvf_get_nxt_sqentry(struct snd_queue *sq, int qentry)
1680 {
1681 qentry++;
1682 qentry &= (sq->dmem.q_len - 1);
1683 return (qentry);
1684 }
1685
1686 static void
nicvf_sq_enable(struct nicvf * nic,struct snd_queue * sq,int qidx)1687 nicvf_sq_enable(struct nicvf *nic, struct snd_queue *sq, int qidx)
1688 {
1689 uint64_t sq_cfg;
1690
1691 sq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_CFG, qidx);
1692 sq_cfg |= NICVF_SQ_EN;
1693 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, sq_cfg);
1694 /* Ring doorbell so that H/W restarts processing SQEs */
1695 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR, qidx, 0);
1696 }
1697
1698 static void
nicvf_sq_disable(struct nicvf * nic,int qidx)1699 nicvf_sq_disable(struct nicvf *nic, int qidx)
1700 {
1701 uint64_t sq_cfg;
1702
1703 sq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_CFG, qidx);
1704 sq_cfg &= ~NICVF_SQ_EN;
1705 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, sq_cfg);
1706 }
1707
1708 static void
nicvf_sq_free_used_descs(struct nicvf * nic,struct snd_queue * sq,int qidx)1709 nicvf_sq_free_used_descs(struct nicvf *nic, struct snd_queue *sq, int qidx)
1710 {
1711 uint64_t head;
1712 struct snd_buff *snd_buff;
1713 struct sq_hdr_subdesc *hdr;
1714
1715 NICVF_TX_LOCK(sq);
1716 head = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_HEAD, qidx) >> 4;
1717 while (sq->head != head) {
1718 hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, sq->head);
1719 if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER) {
1720 nicvf_put_sq_desc(sq, 1);
1721 continue;
1722 }
1723 snd_buff = &sq->snd_buff[sq->head];
1724 if (snd_buff->mbuf != NULL) {
1725 bus_dmamap_unload(sq->snd_buff_dmat, snd_buff->dmap);
1726 m_freem(snd_buff->mbuf);
1727 sq->snd_buff[sq->head].mbuf = NULL;
1728 }
1729 nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1);
1730 }
1731 NICVF_TX_UNLOCK(sq);
1732 }
1733
1734 /*
1735 * Add SQ HEADER subdescriptor.
1736 * First subdescriptor for every send descriptor.
1737 */
1738 static __inline int
nicvf_sq_add_hdr_subdesc(struct snd_queue * sq,int qentry,int subdesc_cnt,struct mbuf * mbuf,int len)1739 nicvf_sq_add_hdr_subdesc(struct snd_queue *sq, int qentry,
1740 int subdesc_cnt, struct mbuf *mbuf, int len)
1741 {
1742 struct nicvf *nic;
1743 struct sq_hdr_subdesc *hdr;
1744 struct ether_vlan_header *eh;
1745 #ifdef INET
1746 struct ip *ip;
1747 #endif
1748 #if defined(INET6) || defined(INET)
1749 struct tcphdr *th;
1750 #endif
1751 #ifdef INET
1752 int iphlen;
1753 #endif
1754 int ehdrlen, poff, proto;
1755 uint16_t etype;
1756
1757 nic = sq->nic;
1758
1759 hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, qentry);
1760 sq->snd_buff[qentry].mbuf = mbuf;
1761
1762 memset(hdr, 0, SND_QUEUE_DESC_SIZE);
1763 hdr->subdesc_type = SQ_DESC_TYPE_HEADER;
1764 /* Enable notification via CQE after processing SQE */
1765 hdr->post_cqe = 1;
1766 /* No of subdescriptors following this */
1767 hdr->subdesc_cnt = subdesc_cnt;
1768 hdr->tot_len = len;
1769
1770 eh = mtod(mbuf, struct ether_vlan_header *);
1771 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
1772 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
1773 etype = ntohs(eh->evl_proto);
1774 } else {
1775 ehdrlen = ETHER_HDR_LEN;
1776 etype = ntohs(eh->evl_encap_proto);
1777 }
1778
1779 poff = proto = -1;
1780 switch (etype) {
1781 #ifdef INET6
1782 case ETHERTYPE_IPV6:
1783 if (mbuf->m_len < ehdrlen + sizeof(struct ip6_hdr)) {
1784 mbuf = m_pullup(mbuf, ehdrlen +sizeof(struct ip6_hdr));
1785 sq->snd_buff[qentry].mbuf = NULL;
1786 if (mbuf == NULL)
1787 return (ENOBUFS);
1788 }
1789 poff = ip6_lasthdr(mbuf, ehdrlen, IPPROTO_IPV6, &proto);
1790 if (poff < 0)
1791 return (ENOBUFS);
1792 poff += ehdrlen;
1793 break;
1794 #endif
1795 #ifdef INET
1796 case ETHERTYPE_IP:
1797 if (mbuf->m_len < ehdrlen + sizeof(struct ip)) {
1798 mbuf = m_pullup(mbuf, ehdrlen + sizeof(struct ip));
1799 sq->snd_buff[qentry].mbuf = mbuf;
1800 if (mbuf == NULL)
1801 return (ENOBUFS);
1802 }
1803 if (mbuf->m_pkthdr.csum_flags & CSUM_IP)
1804 hdr->csum_l3 = 1; /* Enable IP csum calculation */
1805
1806 ip = (struct ip *)(mbuf->m_data + ehdrlen);
1807 iphlen = ip->ip_hl << 2;
1808 poff = ehdrlen + iphlen;
1809 proto = ip->ip_p;
1810 break;
1811 #endif
1812 }
1813
1814 #if defined(INET6) || defined(INET)
1815 if (poff > 0 && mbuf->m_pkthdr.csum_flags != 0) {
1816 switch (proto) {
1817 case IPPROTO_TCP:
1818 if ((mbuf->m_pkthdr.csum_flags & CSUM_TCP) == 0)
1819 break;
1820
1821 if (mbuf->m_len < (poff + sizeof(struct tcphdr))) {
1822 mbuf = m_pullup(mbuf, poff + sizeof(struct tcphdr));
1823 sq->snd_buff[qentry].mbuf = mbuf;
1824 if (mbuf == NULL)
1825 return (ENOBUFS);
1826 }
1827 hdr->csum_l4 = SEND_L4_CSUM_TCP;
1828 break;
1829 case IPPROTO_UDP:
1830 if ((mbuf->m_pkthdr.csum_flags & CSUM_UDP) == 0)
1831 break;
1832
1833 if (mbuf->m_len < (poff + sizeof(struct udphdr))) {
1834 mbuf = m_pullup(mbuf, poff + sizeof(struct udphdr));
1835 sq->snd_buff[qentry].mbuf = mbuf;
1836 if (mbuf == NULL)
1837 return (ENOBUFS);
1838 }
1839 hdr->csum_l4 = SEND_L4_CSUM_UDP;
1840 break;
1841 case IPPROTO_SCTP:
1842 if ((mbuf->m_pkthdr.csum_flags & CSUM_SCTP) == 0)
1843 break;
1844
1845 if (mbuf->m_len < (poff + sizeof(struct sctphdr))) {
1846 mbuf = m_pullup(mbuf, poff + sizeof(struct sctphdr));
1847 sq->snd_buff[qentry].mbuf = mbuf;
1848 if (mbuf == NULL)
1849 return (ENOBUFS);
1850 }
1851 hdr->csum_l4 = SEND_L4_CSUM_SCTP;
1852 break;
1853 default:
1854 break;
1855 }
1856 hdr->l3_offset = ehdrlen;
1857 hdr->l4_offset = poff;
1858 }
1859
1860 if ((mbuf->m_pkthdr.tso_segsz != 0) && nic->hw_tso) {
1861 th = (struct tcphdr *)((caddr_t)(mbuf->m_data + poff));
1862
1863 hdr->tso = 1;
1864 hdr->tso_start = poff + (th->th_off * 4);
1865 hdr->tso_max_paysize = mbuf->m_pkthdr.tso_segsz;
1866 hdr->inner_l3_offset = ehdrlen - 2;
1867 nic->drv_stats.tx_tso++;
1868 }
1869 #endif
1870
1871 return (0);
1872 }
1873
1874 /*
1875 * SQ GATHER subdescriptor
1876 * Must follow HDR descriptor
1877 */
nicvf_sq_add_gather_subdesc(struct snd_queue * sq,int qentry,int size,uint64_t data)1878 static inline void nicvf_sq_add_gather_subdesc(struct snd_queue *sq, int qentry,
1879 int size, uint64_t data)
1880 {
1881 struct sq_gather_subdesc *gather;
1882
1883 qentry &= (sq->dmem.q_len - 1);
1884 gather = (struct sq_gather_subdesc *)GET_SQ_DESC(sq, qentry);
1885
1886 memset(gather, 0, SND_QUEUE_DESC_SIZE);
1887 gather->subdesc_type = SQ_DESC_TYPE_GATHER;
1888 gather->ld_type = NIC_SEND_LD_TYPE_E_LDD;
1889 gather->size = size;
1890 gather->addr = data;
1891 }
1892
1893 /* Put an mbuf to a SQ for packet transfer. */
1894 static int
nicvf_tx_mbuf_locked(struct snd_queue * sq,struct mbuf ** mbufp)1895 nicvf_tx_mbuf_locked(struct snd_queue *sq, struct mbuf **mbufp)
1896 {
1897 bus_dma_segment_t segs[256];
1898 struct snd_buff *snd_buff;
1899 size_t seg;
1900 int nsegs, qentry;
1901 int subdesc_cnt;
1902 int err;
1903
1904 NICVF_TX_LOCK_ASSERT(sq);
1905
1906 if (sq->free_cnt == 0)
1907 return (ENOBUFS);
1908
1909 snd_buff = &sq->snd_buff[sq->tail];
1910
1911 err = bus_dmamap_load_mbuf_sg(sq->snd_buff_dmat, snd_buff->dmap,
1912 *mbufp, segs, &nsegs, BUS_DMA_NOWAIT);
1913 if (__predict_false(err != 0)) {
1914 /* ARM64TODO: Add mbuf defragmenting if we lack maps */
1915 m_freem(*mbufp);
1916 *mbufp = NULL;
1917 return (err);
1918 }
1919
1920 /* Set how many subdescriptors is required */
1921 subdesc_cnt = MIN_SQ_DESC_PER_PKT_XMIT + nsegs - 1;
1922 if (subdesc_cnt > sq->free_cnt) {
1923 /* ARM64TODO: Add mbuf defragmentation if we lack descriptors */
1924 bus_dmamap_unload(sq->snd_buff_dmat, snd_buff->dmap);
1925 return (ENOBUFS);
1926 }
1927
1928 qentry = nicvf_get_sq_desc(sq, subdesc_cnt);
1929
1930 /* Add SQ header subdesc */
1931 err = nicvf_sq_add_hdr_subdesc(sq, qentry, subdesc_cnt - 1, *mbufp,
1932 (*mbufp)->m_pkthdr.len);
1933 if (err != 0) {
1934 nicvf_put_sq_desc(sq, subdesc_cnt);
1935 bus_dmamap_unload(sq->snd_buff_dmat, snd_buff->dmap);
1936 if (err == ENOBUFS) {
1937 m_freem(*mbufp);
1938 *mbufp = NULL;
1939 }
1940 return (err);
1941 }
1942
1943 /* Add SQ gather subdescs */
1944 for (seg = 0; seg < nsegs; seg++) {
1945 qentry = nicvf_get_nxt_sqentry(sq, qentry);
1946 nicvf_sq_add_gather_subdesc(sq, qentry, segs[seg].ds_len,
1947 segs[seg].ds_addr);
1948 }
1949
1950 /* make sure all memory stores are done before ringing doorbell */
1951 bus_dmamap_sync(sq->dmem.dmat, sq->dmem.dmap, BUS_DMASYNC_PREWRITE);
1952
1953 dprintf(sq->nic->dev, "%s: sq->idx: %d, subdesc_cnt: %d\n",
1954 __func__, sq->idx, subdesc_cnt);
1955 /* Inform HW to xmit new packet */
1956 nicvf_queue_reg_write(sq->nic, NIC_QSET_SQ_0_7_DOOR,
1957 sq->idx, subdesc_cnt);
1958 return (0);
1959 }
1960
1961 static __inline u_int
frag_num(u_int i)1962 frag_num(u_int i)
1963 {
1964 #if BYTE_ORDER == BIG_ENDIAN
1965 return ((i & ~3) + 3 - (i & 3));
1966 #else
1967 return (i);
1968 #endif
1969 }
1970
1971 /* Returns MBUF for a received packet */
1972 struct mbuf *
nicvf_get_rcv_mbuf(struct nicvf * nic,struct cqe_rx_t * cqe_rx)1973 nicvf_get_rcv_mbuf(struct nicvf *nic, struct cqe_rx_t *cqe_rx)
1974 {
1975 int frag;
1976 int payload_len = 0;
1977 struct mbuf *mbuf;
1978 struct mbuf *mbuf_frag;
1979 uint16_t *rb_lens = NULL;
1980 uint64_t *rb_ptrs = NULL;
1981
1982 mbuf = NULL;
1983 rb_lens = (uint16_t *)((uint8_t *)cqe_rx + (3 * sizeof(uint64_t)));
1984 rb_ptrs = (uint64_t *)((uint8_t *)cqe_rx + (6 * sizeof(uint64_t)));
1985
1986 dprintf(nic->dev, "%s rb_cnt %d rb0_ptr %lx rb0_sz %d\n",
1987 __func__, cqe_rx->rb_cnt, cqe_rx->rb0_ptr, cqe_rx->rb0_sz);
1988
1989 for (frag = 0; frag < cqe_rx->rb_cnt; frag++) {
1990 payload_len = rb_lens[frag_num(frag)];
1991 if (frag == 0) {
1992 /* First fragment */
1993 mbuf = nicvf_rb_ptr_to_mbuf(nic,
1994 (*rb_ptrs - cqe_rx->align_pad));
1995 mbuf->m_len = payload_len;
1996 mbuf->m_data += cqe_rx->align_pad;
1997 if_setrcvif(mbuf, nic->ifp);
1998 } else {
1999 /* Add fragments */
2000 mbuf_frag = nicvf_rb_ptr_to_mbuf(nic, *rb_ptrs);
2001 m_append(mbuf, payload_len, mbuf_frag->m_data);
2002 m_freem(mbuf_frag);
2003 }
2004 /* Next buffer pointer */
2005 rb_ptrs++;
2006 }
2007
2008 if (__predict_true(mbuf != NULL)) {
2009 m_fixhdr(mbuf);
2010 mbuf->m_pkthdr.flowid = cqe_rx->rq_idx;
2011 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE);
2012 if (__predict_true((if_getcapenable(nic->ifp) & IFCAP_RXCSUM) != 0)) {
2013 /*
2014 * HW by default verifies IP & TCP/UDP/SCTP checksums
2015 */
2016 if (__predict_true(cqe_rx->l3_type == L3TYPE_IPV4)) {
2017 mbuf->m_pkthdr.csum_flags =
2018 (CSUM_IP_CHECKED | CSUM_IP_VALID);
2019 }
2020
2021 switch (cqe_rx->l4_type) {
2022 case L4TYPE_UDP:
2023 case L4TYPE_TCP: /* fall through */
2024 mbuf->m_pkthdr.csum_flags |=
2025 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
2026 mbuf->m_pkthdr.csum_data = 0xffff;
2027 break;
2028 case L4TYPE_SCTP:
2029 mbuf->m_pkthdr.csum_flags |= CSUM_SCTP_VALID;
2030 break;
2031 default:
2032 break;
2033 }
2034 }
2035 }
2036
2037 return (mbuf);
2038 }
2039
2040 /* Enable interrupt */
2041 void
nicvf_enable_intr(struct nicvf * nic,int int_type,int q_idx)2042 nicvf_enable_intr(struct nicvf *nic, int int_type, int q_idx)
2043 {
2044 uint64_t reg_val;
2045
2046 reg_val = nicvf_reg_read(nic, NIC_VF_ENA_W1S);
2047
2048 switch (int_type) {
2049 case NICVF_INTR_CQ:
2050 reg_val |= ((1UL << q_idx) << NICVF_INTR_CQ_SHIFT);
2051 break;
2052 case NICVF_INTR_SQ:
2053 reg_val |= ((1UL << q_idx) << NICVF_INTR_SQ_SHIFT);
2054 break;
2055 case NICVF_INTR_RBDR:
2056 reg_val |= ((1UL << q_idx) << NICVF_INTR_RBDR_SHIFT);
2057 break;
2058 case NICVF_INTR_PKT_DROP:
2059 reg_val |= (1UL << NICVF_INTR_PKT_DROP_SHIFT);
2060 break;
2061 case NICVF_INTR_TCP_TIMER:
2062 reg_val |= (1UL << NICVF_INTR_TCP_TIMER_SHIFT);
2063 break;
2064 case NICVF_INTR_MBOX:
2065 reg_val |= (1UL << NICVF_INTR_MBOX_SHIFT);
2066 break;
2067 case NICVF_INTR_QS_ERR:
2068 reg_val |= (1UL << NICVF_INTR_QS_ERR_SHIFT);
2069 break;
2070 default:
2071 device_printf(nic->dev,
2072 "Failed to enable interrupt: unknown type\n");
2073 break;
2074 }
2075
2076 nicvf_reg_write(nic, NIC_VF_ENA_W1S, reg_val);
2077 }
2078
2079 /* Disable interrupt */
2080 void
nicvf_disable_intr(struct nicvf * nic,int int_type,int q_idx)2081 nicvf_disable_intr(struct nicvf *nic, int int_type, int q_idx)
2082 {
2083 uint64_t reg_val = 0;
2084
2085 switch (int_type) {
2086 case NICVF_INTR_CQ:
2087 reg_val |= ((1UL << q_idx) << NICVF_INTR_CQ_SHIFT);
2088 break;
2089 case NICVF_INTR_SQ:
2090 reg_val |= ((1UL << q_idx) << NICVF_INTR_SQ_SHIFT);
2091 break;
2092 case NICVF_INTR_RBDR:
2093 reg_val |= ((1UL << q_idx) << NICVF_INTR_RBDR_SHIFT);
2094 break;
2095 case NICVF_INTR_PKT_DROP:
2096 reg_val |= (1UL << NICVF_INTR_PKT_DROP_SHIFT);
2097 break;
2098 case NICVF_INTR_TCP_TIMER:
2099 reg_val |= (1UL << NICVF_INTR_TCP_TIMER_SHIFT);
2100 break;
2101 case NICVF_INTR_MBOX:
2102 reg_val |= (1UL << NICVF_INTR_MBOX_SHIFT);
2103 break;
2104 case NICVF_INTR_QS_ERR:
2105 reg_val |= (1UL << NICVF_INTR_QS_ERR_SHIFT);
2106 break;
2107 default:
2108 device_printf(nic->dev,
2109 "Failed to disable interrupt: unknown type\n");
2110 break;
2111 }
2112
2113 nicvf_reg_write(nic, NIC_VF_ENA_W1C, reg_val);
2114 }
2115
2116 /* Clear interrupt */
2117 void
nicvf_clear_intr(struct nicvf * nic,int int_type,int q_idx)2118 nicvf_clear_intr(struct nicvf *nic, int int_type, int q_idx)
2119 {
2120 uint64_t reg_val = 0;
2121
2122 switch (int_type) {
2123 case NICVF_INTR_CQ:
2124 reg_val = ((1UL << q_idx) << NICVF_INTR_CQ_SHIFT);
2125 break;
2126 case NICVF_INTR_SQ:
2127 reg_val = ((1UL << q_idx) << NICVF_INTR_SQ_SHIFT);
2128 break;
2129 case NICVF_INTR_RBDR:
2130 reg_val = ((1UL << q_idx) << NICVF_INTR_RBDR_SHIFT);
2131 break;
2132 case NICVF_INTR_PKT_DROP:
2133 reg_val = (1UL << NICVF_INTR_PKT_DROP_SHIFT);
2134 break;
2135 case NICVF_INTR_TCP_TIMER:
2136 reg_val = (1UL << NICVF_INTR_TCP_TIMER_SHIFT);
2137 break;
2138 case NICVF_INTR_MBOX:
2139 reg_val = (1UL << NICVF_INTR_MBOX_SHIFT);
2140 break;
2141 case NICVF_INTR_QS_ERR:
2142 reg_val |= (1UL << NICVF_INTR_QS_ERR_SHIFT);
2143 break;
2144 default:
2145 device_printf(nic->dev,
2146 "Failed to clear interrupt: unknown type\n");
2147 break;
2148 }
2149
2150 nicvf_reg_write(nic, NIC_VF_INT, reg_val);
2151 }
2152
2153 /* Check if interrupt is enabled */
2154 int
nicvf_is_intr_enabled(struct nicvf * nic,int int_type,int q_idx)2155 nicvf_is_intr_enabled(struct nicvf *nic, int int_type, int q_idx)
2156 {
2157 uint64_t reg_val;
2158 uint64_t mask = 0xff;
2159
2160 reg_val = nicvf_reg_read(nic, NIC_VF_ENA_W1S);
2161
2162 switch (int_type) {
2163 case NICVF_INTR_CQ:
2164 mask = ((1UL << q_idx) << NICVF_INTR_CQ_SHIFT);
2165 break;
2166 case NICVF_INTR_SQ:
2167 mask = ((1UL << q_idx) << NICVF_INTR_SQ_SHIFT);
2168 break;
2169 case NICVF_INTR_RBDR:
2170 mask = ((1UL << q_idx) << NICVF_INTR_RBDR_SHIFT);
2171 break;
2172 case NICVF_INTR_PKT_DROP:
2173 mask = NICVF_INTR_PKT_DROP_MASK;
2174 break;
2175 case NICVF_INTR_TCP_TIMER:
2176 mask = NICVF_INTR_TCP_TIMER_MASK;
2177 break;
2178 case NICVF_INTR_MBOX:
2179 mask = NICVF_INTR_MBOX_MASK;
2180 break;
2181 case NICVF_INTR_QS_ERR:
2182 mask = NICVF_INTR_QS_ERR_MASK;
2183 break;
2184 default:
2185 device_printf(nic->dev,
2186 "Failed to check interrupt enable: unknown type\n");
2187 break;
2188 }
2189
2190 return (reg_val & mask);
2191 }
2192
2193 void
nicvf_update_rq_stats(struct nicvf * nic,int rq_idx)2194 nicvf_update_rq_stats(struct nicvf *nic, int rq_idx)
2195 {
2196 struct rcv_queue *rq;
2197
2198 #define GET_RQ_STATS(reg) \
2199 nicvf_reg_read(nic, NIC_QSET_RQ_0_7_STAT_0_1 |\
2200 (rq_idx << NIC_Q_NUM_SHIFT) | (reg << 3))
2201
2202 rq = &nic->qs->rq[rq_idx];
2203 rq->stats.bytes = GET_RQ_STATS(RQ_SQ_STATS_OCTS);
2204 rq->stats.pkts = GET_RQ_STATS(RQ_SQ_STATS_PKTS);
2205 }
2206
2207 void
nicvf_update_sq_stats(struct nicvf * nic,int sq_idx)2208 nicvf_update_sq_stats(struct nicvf *nic, int sq_idx)
2209 {
2210 struct snd_queue *sq;
2211
2212 #define GET_SQ_STATS(reg) \
2213 nicvf_reg_read(nic, NIC_QSET_SQ_0_7_STAT_0_1 |\
2214 (sq_idx << NIC_Q_NUM_SHIFT) | (reg << 3))
2215
2216 sq = &nic->qs->sq[sq_idx];
2217 sq->stats.bytes = GET_SQ_STATS(RQ_SQ_STATS_OCTS);
2218 sq->stats.pkts = GET_SQ_STATS(RQ_SQ_STATS_PKTS);
2219 }
2220
2221 /* Check for errors in the receive cmp.queue entry */
2222 int
nicvf_check_cqe_rx_errs(struct nicvf * nic,struct cmp_queue * cq,struct cqe_rx_t * cqe_rx)2223 nicvf_check_cqe_rx_errs(struct nicvf *nic, struct cmp_queue *cq,
2224 struct cqe_rx_t *cqe_rx)
2225 {
2226 struct nicvf_hw_stats *stats = &nic->hw_stats;
2227 struct nicvf_drv_stats *drv_stats = &nic->drv_stats;
2228
2229 if (!cqe_rx->err_level && !cqe_rx->err_opcode) {
2230 drv_stats->rx_frames_ok++;
2231 return (0);
2232 }
2233
2234 switch (cqe_rx->err_opcode) {
2235 case CQ_RX_ERROP_RE_PARTIAL:
2236 stats->rx_bgx_truncated_pkts++;
2237 break;
2238 case CQ_RX_ERROP_RE_JABBER:
2239 stats->rx_jabber_errs++;
2240 break;
2241 case CQ_RX_ERROP_RE_FCS:
2242 stats->rx_fcs_errs++;
2243 break;
2244 case CQ_RX_ERROP_RE_RX_CTL:
2245 stats->rx_bgx_errs++;
2246 break;
2247 case CQ_RX_ERROP_PREL2_ERR:
2248 stats->rx_prel2_errs++;
2249 break;
2250 case CQ_RX_ERROP_L2_MAL:
2251 stats->rx_l2_hdr_malformed++;
2252 break;
2253 case CQ_RX_ERROP_L2_OVERSIZE:
2254 stats->rx_oversize++;
2255 break;
2256 case CQ_RX_ERROP_L2_UNDERSIZE:
2257 stats->rx_undersize++;
2258 break;
2259 case CQ_RX_ERROP_L2_LENMISM:
2260 stats->rx_l2_len_mismatch++;
2261 break;
2262 case CQ_RX_ERROP_L2_PCLP:
2263 stats->rx_l2_pclp++;
2264 break;
2265 case CQ_RX_ERROP_IP_NOT:
2266 stats->rx_ip_ver_errs++;
2267 break;
2268 case CQ_RX_ERROP_IP_CSUM_ERR:
2269 stats->rx_ip_csum_errs++;
2270 break;
2271 case CQ_RX_ERROP_IP_MAL:
2272 stats->rx_ip_hdr_malformed++;
2273 break;
2274 case CQ_RX_ERROP_IP_MALD:
2275 stats->rx_ip_payload_malformed++;
2276 break;
2277 case CQ_RX_ERROP_IP_HOP:
2278 stats->rx_ip_ttl_errs++;
2279 break;
2280 case CQ_RX_ERROP_L3_PCLP:
2281 stats->rx_l3_pclp++;
2282 break;
2283 case CQ_RX_ERROP_L4_MAL:
2284 stats->rx_l4_malformed++;
2285 break;
2286 case CQ_RX_ERROP_L4_CHK:
2287 stats->rx_l4_csum_errs++;
2288 break;
2289 case CQ_RX_ERROP_UDP_LEN:
2290 stats->rx_udp_len_errs++;
2291 break;
2292 case CQ_RX_ERROP_L4_PORT:
2293 stats->rx_l4_port_errs++;
2294 break;
2295 case CQ_RX_ERROP_TCP_FLAG:
2296 stats->rx_tcp_flag_errs++;
2297 break;
2298 case CQ_RX_ERROP_TCP_OFFSET:
2299 stats->rx_tcp_offset_errs++;
2300 break;
2301 case CQ_RX_ERROP_L4_PCLP:
2302 stats->rx_l4_pclp++;
2303 break;
2304 case CQ_RX_ERROP_RBDR_TRUNC:
2305 stats->rx_truncated_pkts++;
2306 break;
2307 }
2308
2309 return (1);
2310 }
2311
2312 /* Check for errors in the send cmp.queue entry */
2313 int
nicvf_check_cqe_tx_errs(struct nicvf * nic,struct cmp_queue * cq,struct cqe_send_t * cqe_tx)2314 nicvf_check_cqe_tx_errs(struct nicvf *nic, struct cmp_queue *cq,
2315 struct cqe_send_t *cqe_tx)
2316 {
2317 struct cmp_queue_stats *stats = &cq->stats;
2318
2319 switch (cqe_tx->send_status) {
2320 case CQ_TX_ERROP_GOOD:
2321 stats->tx.good++;
2322 return (0);
2323 case CQ_TX_ERROP_DESC_FAULT:
2324 stats->tx.desc_fault++;
2325 break;
2326 case CQ_TX_ERROP_HDR_CONS_ERR:
2327 stats->tx.hdr_cons_err++;
2328 break;
2329 case CQ_TX_ERROP_SUBDC_ERR:
2330 stats->tx.subdesc_err++;
2331 break;
2332 case CQ_TX_ERROP_IMM_SIZE_OFLOW:
2333 stats->tx.imm_size_oflow++;
2334 break;
2335 case CQ_TX_ERROP_DATA_SEQUENCE_ERR:
2336 stats->tx.data_seq_err++;
2337 break;
2338 case CQ_TX_ERROP_MEM_SEQUENCE_ERR:
2339 stats->tx.mem_seq_err++;
2340 break;
2341 case CQ_TX_ERROP_LOCK_VIOL:
2342 stats->tx.lock_viol++;
2343 break;
2344 case CQ_TX_ERROP_DATA_FAULT:
2345 stats->tx.data_fault++;
2346 break;
2347 case CQ_TX_ERROP_TSTMP_CONFLICT:
2348 stats->tx.tstmp_conflict++;
2349 break;
2350 case CQ_TX_ERROP_TSTMP_TIMEOUT:
2351 stats->tx.tstmp_timeout++;
2352 break;
2353 case CQ_TX_ERROP_MEM_FAULT:
2354 stats->tx.mem_fault++;
2355 break;
2356 case CQ_TX_ERROP_CK_OVERLAP:
2357 stats->tx.csum_overlap++;
2358 break;
2359 case CQ_TX_ERROP_CK_OFLOW:
2360 stats->tx.csum_overflow++;
2361 break;
2362 }
2363
2364 return (1);
2365 }
2366