1 /*
2 * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 */
33
34 #include <linux/page.h>
35 #include <linux/mlx4/cq.h>
36 #include <linux/slab.h>
37 #include <linux/mlx4/qp.h>
38 #include <linux/if_vlan.h>
39 #include <linux/vmalloc.h>
40 #include <linux/moduleparam.h>
41
42 #include <netinet/in_systm.h>
43 #include <netinet/in.h>
44 #include <netinet/if_ether.h>
45 #include <netinet/ip.h>
46 #include <netinet/ip6.h>
47 #include <netinet/tcp.h>
48 #include <netinet/tcp_lro.h>
49 #include <netinet/udp.h>
50
51 #include "mlx4_en.h"
52
53 enum {
54 MAX_INLINE = 104, /* 128 - 16 - 4 - 4 */
55 MAX_BF = 256,
56 MIN_PKT_LEN = 17,
57 };
58
59 static int inline_thold __read_mostly = MAX_INLINE;
60
61 module_param_named(inline_thold, inline_thold, uint, 0444);
62 MODULE_PARM_DESC(inline_thold, "threshold for using inline data");
63
mlx4_en_create_tx_ring(struct mlx4_en_priv * priv,struct mlx4_en_tx_ring ** pring,u32 size,u16 stride,int node,int queue_idx)64 int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
65 struct mlx4_en_tx_ring **pring, u32 size,
66 u16 stride, int node, int queue_idx)
67 {
68 struct mlx4_en_dev *mdev = priv->mdev;
69 struct mlx4_en_tx_ring *ring;
70 uint32_t x;
71 int tmp;
72 int err;
73
74 ring = kzalloc_node(sizeof(struct mlx4_en_tx_ring), GFP_KERNEL, node);
75 if (!ring) {
76 ring = kzalloc(sizeof(struct mlx4_en_tx_ring), GFP_KERNEL);
77 if (!ring) {
78 en_err(priv, "Failed allocating TX ring\n");
79 return -ENOMEM;
80 }
81 }
82
83 /* Create DMA descriptor TAG */
84 if ((err = -bus_dma_tag_create(
85 bus_get_dma_tag(mdev->pdev->dev.bsddev),
86 1, /* any alignment */
87 0, /* no boundary */
88 BUS_SPACE_MAXADDR, /* lowaddr */
89 BUS_SPACE_MAXADDR, /* highaddr */
90 NULL, NULL, /* filter, filterarg */
91 MLX4_EN_TX_MAX_PAYLOAD_SIZE, /* maxsize */
92 MLX4_EN_TX_MAX_MBUF_FRAGS, /* nsegments */
93 MLX4_EN_TX_MAX_MBUF_SIZE, /* maxsegsize */
94 0, /* flags */
95 NULL, NULL, /* lockfunc, lockfuncarg */
96 &ring->dma_tag)))
97 goto done;
98
99 ring->size = size;
100 ring->size_mask = size - 1;
101 ring->stride = stride;
102 ring->inline_thold = MAX(MIN_PKT_LEN, MIN(inline_thold, MAX_INLINE));
103 mtx_init(&ring->tx_lock.m, "mlx4 tx", NULL, MTX_DEF);
104 mtx_init(&ring->comp_lock.m, "mlx4 comp", NULL, MTX_DEF);
105
106 /* Allocate the buf ring */
107 ring->br = buf_ring_alloc(MLX4_EN_DEF_TX_QUEUE_SIZE, M_DEVBUF,
108 M_WAITOK, &ring->tx_lock.m);
109 if (ring->br == NULL) {
110 en_err(priv, "Failed allocating tx_info ring\n");
111 err = -ENOMEM;
112 goto err_free_dma_tag;
113 }
114
115 tmp = size * sizeof(struct mlx4_en_tx_info);
116 ring->tx_info = kzalloc_node(tmp, GFP_KERNEL, node);
117 if (!ring->tx_info) {
118 ring->tx_info = kzalloc(tmp, GFP_KERNEL);
119 if (!ring->tx_info) {
120 err = -ENOMEM;
121 goto err_ring;
122 }
123 }
124
125 /* Create DMA descriptor MAPs */
126 for (x = 0; x != size; x++) {
127 err = -bus_dmamap_create(ring->dma_tag, 0,
128 &ring->tx_info[x].dma_map);
129 if (err != 0) {
130 while (x--) {
131 bus_dmamap_destroy(ring->dma_tag,
132 ring->tx_info[x].dma_map);
133 }
134 goto err_info;
135 }
136 }
137
138 en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n",
139 ring->tx_info, tmp);
140
141 ring->buf_size = ALIGN(size * ring->stride, MLX4_EN_PAGE_SIZE);
142
143 /* Allocate HW buffers on provided NUMA node */
144 err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size,
145 2 * PAGE_SIZE);
146 if (err) {
147 en_err(priv, "Failed allocating hwq resources\n");
148 goto err_dma_map;
149 }
150
151 err = mlx4_en_map_buffer(&ring->wqres.buf);
152 if (err) {
153 en_err(priv, "Failed to map TX buffer\n");
154 goto err_hwq_res;
155 }
156
157 ring->buf = ring->wqres.buf.direct.buf;
158
159 en_dbg(DRV, priv, "Allocated TX ring (addr:%p) - buf:%p size:%d "
160 "buf_size:%d dma:%llx\n", ring, ring->buf, ring->size,
161 ring->buf_size, (unsigned long long) ring->wqres.buf.direct.map);
162
163 err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn,
164 MLX4_RESERVE_BF_QP);
165 if (err) {
166 en_err(priv, "failed reserving qp for TX ring\n");
167 goto err_map;
168 }
169
170 err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->qp);
171 if (err) {
172 en_err(priv, "Failed allocating qp %d\n", ring->qpn);
173 goto err_reserve;
174 }
175 ring->qp.event = mlx4_en_sqp_event;
176
177 err = mlx4_bf_alloc(mdev->dev, &ring->bf, node);
178 if (err) {
179 en_dbg(DRV, priv, "working without blueflame (%d)", err);
180 ring->bf.uar = &mdev->priv_uar;
181 ring->bf.uar->map = mdev->uar_map;
182 ring->bf_enabled = false;
183 } else
184 ring->bf_enabled = true;
185 ring->queue_index = queue_idx;
186 if (queue_idx < priv->num_tx_rings_p_up )
187 CPU_SET(queue_idx, &ring->affinity_mask);
188
189 *pring = ring;
190 return 0;
191
192 err_reserve:
193 mlx4_qp_release_range(mdev->dev, ring->qpn, 1);
194 err_map:
195 mlx4_en_unmap_buffer(&ring->wqres.buf);
196 err_hwq_res:
197 mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
198 err_dma_map:
199 for (x = 0; x != size; x++)
200 bus_dmamap_destroy(ring->dma_tag, ring->tx_info[x].dma_map);
201 err_info:
202 vfree(ring->tx_info);
203 err_ring:
204 buf_ring_free(ring->br, M_DEVBUF);
205 err_free_dma_tag:
206 bus_dma_tag_destroy(ring->dma_tag);
207 done:
208 kfree(ring);
209 return err;
210 }
211
mlx4_en_destroy_tx_ring(struct mlx4_en_priv * priv,struct mlx4_en_tx_ring ** pring)212 void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv,
213 struct mlx4_en_tx_ring **pring)
214 {
215 struct mlx4_en_dev *mdev = priv->mdev;
216 struct mlx4_en_tx_ring *ring = *pring;
217 uint32_t x;
218 en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn);
219
220 buf_ring_free(ring->br, M_DEVBUF);
221 if (ring->bf_enabled)
222 mlx4_bf_free(mdev->dev, &ring->bf);
223 mlx4_qp_remove(mdev->dev, &ring->qp);
224 mlx4_qp_free(mdev->dev, &ring->qp);
225 mlx4_qp_release_range(priv->mdev->dev, ring->qpn, 1);
226 mlx4_en_unmap_buffer(&ring->wqres.buf);
227 mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
228 for (x = 0; x != ring->size; x++)
229 bus_dmamap_destroy(ring->dma_tag, ring->tx_info[x].dma_map);
230 vfree(ring->tx_info);
231 mtx_destroy(&ring->tx_lock.m);
232 mtx_destroy(&ring->comp_lock.m);
233 bus_dma_tag_destroy(ring->dma_tag);
234 kfree(ring);
235 *pring = NULL;
236 }
237
mlx4_en_activate_tx_ring(struct mlx4_en_priv * priv,struct mlx4_en_tx_ring * ring,int cq,int user_prio)238 int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv,
239 struct mlx4_en_tx_ring *ring,
240 int cq, int user_prio)
241 {
242 struct mlx4_en_dev *mdev = priv->mdev;
243 int err;
244
245 ring->cqn = cq;
246 ring->prod = 0;
247 ring->cons = 0xffffffff;
248 ring->last_nr_txbb = 1;
249 ring->poll_cnt = 0;
250 ring->blocked = 0;
251 memset(ring->buf, 0, ring->buf_size);
252
253 ring->qp_state = MLX4_QP_STATE_RST;
254 ring->doorbell_qpn = ring->qp.qpn << 8;
255
256 mlx4_en_fill_qp_context(priv, ring->size, ring->stride, 1, 0, ring->qpn,
257 ring->cqn, user_prio, &ring->context);
258 if (ring->bf_enabled)
259 ring->context.usr_page = cpu_to_be32(ring->bf.uar->index);
260
261 err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, &ring->context,
262 &ring->qp, &ring->qp_state);
263 return err;
264 }
265
mlx4_en_deactivate_tx_ring(struct mlx4_en_priv * priv,struct mlx4_en_tx_ring * ring)266 void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv,
267 struct mlx4_en_tx_ring *ring)
268 {
269 struct mlx4_en_dev *mdev = priv->mdev;
270
271 mlx4_qp_modify(mdev->dev, NULL, ring->qp_state,
272 MLX4_QP_STATE_RST, NULL, 0, 0, &ring->qp);
273 }
274
275 static volatile struct mlx4_wqe_data_seg *
mlx4_en_store_inline_lso_data(volatile struct mlx4_wqe_data_seg * dseg,struct mbuf * mb,int len,__be32 owner_bit)276 mlx4_en_store_inline_lso_data(volatile struct mlx4_wqe_data_seg *dseg,
277 struct mbuf *mb, int len, __be32 owner_bit)
278 {
279 uint8_t *inl = __DEVOLATILE(uint8_t *, dseg);
280
281 /* copy data into place */
282 m_copydata(mb, 0, len, inl + 4);
283 dseg += DIV_ROUND_UP(4 + len, DS_SIZE_ALIGNMENT);
284 return (dseg);
285 }
286
287 static void
mlx4_en_store_inline_lso_header(volatile struct mlx4_wqe_data_seg * dseg,int len,__be32 owner_bit)288 mlx4_en_store_inline_lso_header(volatile struct mlx4_wqe_data_seg *dseg,
289 int len, __be32 owner_bit)
290 {
291 }
292
293 static void
mlx4_en_stamp_wqe(struct mlx4_en_priv * priv,struct mlx4_en_tx_ring * ring,u32 index,u8 owner)294 mlx4_en_stamp_wqe(struct mlx4_en_priv *priv,
295 struct mlx4_en_tx_ring *ring, u32 index, u8 owner)
296 {
297 struct mlx4_en_tx_info *tx_info = &ring->tx_info[index];
298 struct mlx4_en_tx_desc *tx_desc = (struct mlx4_en_tx_desc *)
299 (ring->buf + (index * TXBB_SIZE));
300 volatile __be32 *ptr = (__be32 *)tx_desc;
301 const __be32 stamp = cpu_to_be32(STAMP_VAL |
302 ((u32)owner << STAMP_SHIFT));
303 u32 i;
304
305 /* Stamp the freed descriptor */
306 for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE; i += STAMP_STRIDE) {
307 *ptr = stamp;
308 ptr += STAMP_DWORDS;
309 }
310 }
311
312 static u32
mlx4_en_free_tx_desc(struct mlx4_en_priv * priv,struct mlx4_en_tx_ring * ring,u32 index)313 mlx4_en_free_tx_desc(struct mlx4_en_priv *priv,
314 struct mlx4_en_tx_ring *ring, u32 index)
315 {
316 struct mlx4_en_tx_info *tx_info;
317 struct mbuf *mb;
318
319 tx_info = &ring->tx_info[index];
320 mb = tx_info->mb;
321
322 if (mb == NULL)
323 goto done;
324
325 bus_dmamap_sync(ring->dma_tag, tx_info->dma_map,
326 BUS_DMASYNC_POSTWRITE);
327 bus_dmamap_unload(ring->dma_tag, tx_info->dma_map);
328
329 m_freem(mb);
330 done:
331 return (tx_info->nr_txbb);
332 }
333
mlx4_en_free_tx_buf(struct net_device * dev,struct mlx4_en_tx_ring * ring)334 int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring)
335 {
336 struct mlx4_en_priv *priv = netdev_priv(dev);
337 int cnt = 0;
338
339 /* Skip last polled descriptor */
340 ring->cons += ring->last_nr_txbb;
341 en_dbg(DRV, priv, "Freeing Tx buf - cons:0x%x prod:0x%x\n",
342 ring->cons, ring->prod);
343
344 if ((u32) (ring->prod - ring->cons) > ring->size) {
345 en_warn(priv, "Tx consumer passed producer!\n");
346 return 0;
347 }
348
349 while (ring->cons != ring->prod) {
350 ring->last_nr_txbb = mlx4_en_free_tx_desc(priv, ring,
351 ring->cons & ring->size_mask);
352 ring->cons += ring->last_nr_txbb;
353 cnt++;
354 }
355
356 if (cnt)
357 en_dbg(DRV, priv, "Freed %d uncompleted tx descriptors\n", cnt);
358
359 return cnt;
360 }
361
362 static bool
mlx4_en_tx_ring_is_full(struct mlx4_en_tx_ring * ring)363 mlx4_en_tx_ring_is_full(struct mlx4_en_tx_ring *ring)
364 {
365 int wqs;
366 wqs = ring->size - (ring->prod - ring->cons);
367 return (wqs < (HEADROOM + (2 * MLX4_EN_TX_WQE_MAX_WQEBBS)));
368 }
369
mlx4_en_process_tx_cq(struct net_device * dev,struct mlx4_en_cq * cq)370 static int mlx4_en_process_tx_cq(struct net_device *dev,
371 struct mlx4_en_cq *cq)
372 {
373 struct mlx4_en_priv *priv = netdev_priv(dev);
374 struct mlx4_cq *mcq = &cq->mcq;
375 struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->ring];
376 struct mlx4_cqe *cqe;
377 u16 index;
378 u16 new_index, ring_index, stamp_index;
379 u32 txbbs_skipped = 0;
380 u32 txbbs_stamp = 0;
381 u32 cons_index = mcq->cons_index;
382 int size = cq->size;
383 u32 size_mask = ring->size_mask;
384 struct mlx4_cqe *buf = cq->buf;
385 int factor = priv->cqe_factor;
386
387 if (!priv->port_up)
388 return 0;
389
390 index = cons_index & size_mask;
391 cqe = &buf[(index << factor) + factor];
392 ring_index = ring->cons & size_mask;
393 stamp_index = ring_index;
394
395 /* Process all completed CQEs */
396 while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
397 cons_index & size)) {
398 /*
399 * make sure we read the CQE after we read the
400 * ownership bit
401 */
402 rmb();
403
404 if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
405 MLX4_CQE_OPCODE_ERROR)) {
406 en_err(priv, "CQE completed in error - vendor syndrom: 0x%x syndrom: 0x%x\n",
407 ((struct mlx4_err_cqe *)cqe)->
408 vendor_err_syndrome,
409 ((struct mlx4_err_cqe *)cqe)->syndrome);
410 }
411
412 /* Skip over last polled CQE */
413 new_index = be16_to_cpu(cqe->wqe_index) & size_mask;
414
415 do {
416 txbbs_skipped += ring->last_nr_txbb;
417 ring_index = (ring_index + ring->last_nr_txbb) & size_mask;
418 /* free next descriptor */
419 ring->last_nr_txbb = mlx4_en_free_tx_desc(
420 priv, ring, ring_index);
421 mlx4_en_stamp_wqe(priv, ring, stamp_index,
422 !!((ring->cons + txbbs_stamp) &
423 ring->size));
424 stamp_index = ring_index;
425 txbbs_stamp = txbbs_skipped;
426 } while (ring_index != new_index);
427
428 ++cons_index;
429 index = cons_index & size_mask;
430 cqe = &buf[(index << factor) + factor];
431 }
432
433
434 /*
435 * To prevent CQ overflow we first update CQ consumer and only then
436 * the ring consumer.
437 */
438 mcq->cons_index = cons_index;
439 mlx4_cq_set_ci(mcq);
440 wmb();
441 ring->cons += txbbs_skipped;
442
443 /* Wakeup Tx queue if it was stopped and ring is not full */
444 if (unlikely(ring->blocked) && !mlx4_en_tx_ring_is_full(ring)) {
445 ring->blocked = 0;
446 if (atomic_fetchadd_int(&priv->blocked, -1) == 1)
447 atomic_clear_int(&dev->if_drv_flags ,IFF_DRV_OACTIVE);
448 ring->wake_queue++;
449 priv->port_stats.wake_queue++;
450 }
451 return (0);
452 }
453
mlx4_en_tx_irq(struct mlx4_cq * mcq)454 void mlx4_en_tx_irq(struct mlx4_cq *mcq)
455 {
456 struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq);
457 struct mlx4_en_priv *priv = netdev_priv(cq->dev);
458 struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->ring];
459
460 if (!spin_trylock(&ring->comp_lock))
461 return;
462 mlx4_en_process_tx_cq(cq->dev, cq);
463 mod_timer(&cq->timer, jiffies + 1);
464 spin_unlock(&ring->comp_lock);
465 }
466
mlx4_en_poll_tx_cq(unsigned long data)467 void mlx4_en_poll_tx_cq(unsigned long data)
468 {
469 struct mlx4_en_cq *cq = (struct mlx4_en_cq *) data;
470 struct mlx4_en_priv *priv = netdev_priv(cq->dev);
471 struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->ring];
472 u32 inflight;
473
474 INC_PERF_COUNTER(priv->pstats.tx_poll);
475
476 if (!spin_trylock(&ring->comp_lock)) {
477 mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT);
478 return;
479 }
480 mlx4_en_process_tx_cq(cq->dev, cq);
481 inflight = (u32) (ring->prod - ring->cons - ring->last_nr_txbb);
482
483 /* If there are still packets in flight and the timer has not already
484 * been scheduled by the Tx routine then schedule it here to guarantee
485 * completion processing of these packets */
486 if (inflight && priv->port_up)
487 mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT);
488
489 spin_unlock(&ring->comp_lock);
490 }
491
mlx4_en_xmit_poll(struct mlx4_en_priv * priv,int tx_ind)492 static inline void mlx4_en_xmit_poll(struct mlx4_en_priv *priv, int tx_ind)
493 {
494 struct mlx4_en_cq *cq = priv->tx_cq[tx_ind];
495 struct mlx4_en_tx_ring *ring = priv->tx_ring[tx_ind];
496
497 /* If we don't have a pending timer, set one up to catch our recent
498 post in case the interface becomes idle */
499 if (!timer_pending(&cq->timer))
500 mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT);
501
502 /* Poll the CQ every mlx4_en_TX_MODER_POLL packets */
503 if ((++ring->poll_cnt & (MLX4_EN_TX_POLL_MODER - 1)) == 0)
504 if (spin_trylock(&ring->comp_lock)) {
505 mlx4_en_process_tx_cq(priv->dev, cq);
506 spin_unlock(&ring->comp_lock);
507 }
508 }
509
510 static u16
mlx4_en_get_inline_hdr_size(struct mlx4_en_tx_ring * ring,struct mbuf * mb)511 mlx4_en_get_inline_hdr_size(struct mlx4_en_tx_ring *ring, struct mbuf *mb)
512 {
513 u16 retval;
514
515 /* only copy from first fragment, if possible */
516 retval = MIN(ring->inline_thold, mb->m_len);
517
518 /* check for too little data */
519 if (unlikely(retval < MIN_PKT_LEN))
520 retval = MIN(ring->inline_thold, mb->m_pkthdr.len);
521 return (retval);
522 }
523
524 static int
mlx4_en_get_header_size(struct mbuf * mb)525 mlx4_en_get_header_size(struct mbuf *mb)
526 {
527 struct ether_vlan_header *eh;
528 struct tcphdr *th;
529 struct ip *ip;
530 int ip_hlen, tcp_hlen;
531 struct ip6_hdr *ip6;
532 uint16_t eth_type;
533 int eth_hdr_len;
534
535 eh = mtod(mb, struct ether_vlan_header *);
536 if (mb->m_len < ETHER_HDR_LEN)
537 return (0);
538 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
539 eth_type = ntohs(eh->evl_proto);
540 eth_hdr_len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
541 } else {
542 eth_type = ntohs(eh->evl_encap_proto);
543 eth_hdr_len = ETHER_HDR_LEN;
544 }
545 if (mb->m_len < eth_hdr_len)
546 return (0);
547 switch (eth_type) {
548 case ETHERTYPE_IP:
549 ip = (struct ip *)(mb->m_data + eth_hdr_len);
550 if (mb->m_len < eth_hdr_len + sizeof(*ip))
551 return (0);
552 if (ip->ip_p != IPPROTO_TCP)
553 return (0);
554 ip_hlen = ip->ip_hl << 2;
555 eth_hdr_len += ip_hlen;
556 break;
557 case ETHERTYPE_IPV6:
558 ip6 = (struct ip6_hdr *)(mb->m_data + eth_hdr_len);
559 if (mb->m_len < eth_hdr_len + sizeof(*ip6))
560 return (0);
561 if (ip6->ip6_nxt != IPPROTO_TCP)
562 return (0);
563 eth_hdr_len += sizeof(*ip6);
564 break;
565 default:
566 return (0);
567 }
568 if (mb->m_len < eth_hdr_len + sizeof(*th))
569 return (0);
570 th = (struct tcphdr *)(mb->m_data + eth_hdr_len);
571 tcp_hlen = th->th_off << 2;
572 eth_hdr_len += tcp_hlen;
573 if (mb->m_len < eth_hdr_len)
574 return (0);
575 return (eth_hdr_len);
576 }
577
578 static volatile struct mlx4_wqe_data_seg *
mlx4_en_store_inline_data(volatile struct mlx4_wqe_data_seg * dseg,struct mbuf * mb,int len,__be32 owner_bit)579 mlx4_en_store_inline_data(volatile struct mlx4_wqe_data_seg *dseg,
580 struct mbuf *mb, int len, __be32 owner_bit)
581 {
582 uint8_t *inl = __DEVOLATILE(uint8_t *, dseg);
583 const int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - 4;
584
585 if (unlikely(len < MIN_PKT_LEN)) {
586 m_copydata(mb, 0, len, inl + 4);
587 memset(inl + 4 + len, 0, MIN_PKT_LEN - len);
588 dseg += DIV_ROUND_UP(4 + MIN_PKT_LEN, DS_SIZE_ALIGNMENT);
589 } else if (len <= spc) {
590 m_copydata(mb, 0, len, inl + 4);
591 dseg += DIV_ROUND_UP(4 + len, DS_SIZE_ALIGNMENT);
592 } else {
593 m_copydata(mb, 0, spc, inl + 4);
594 m_copydata(mb, spc, len - spc, inl + 8 + spc);
595 dseg += DIV_ROUND_UP(8 + len, DS_SIZE_ALIGNMENT);
596 }
597 return (dseg);
598 }
599
600 static void
mlx4_en_store_inline_header(volatile struct mlx4_wqe_data_seg * dseg,int len,__be32 owner_bit)601 mlx4_en_store_inline_header(volatile struct mlx4_wqe_data_seg *dseg,
602 int len, __be32 owner_bit)
603 {
604 uint8_t *inl = __DEVOLATILE(uint8_t *, dseg);
605 const int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - 4;
606
607 if (unlikely(len < MIN_PKT_LEN)) {
608 *(volatile uint32_t *)inl =
609 SET_BYTE_COUNT((1 << 31) | MIN_PKT_LEN);
610 } else if (len <= spc) {
611 *(volatile uint32_t *)inl =
612 SET_BYTE_COUNT((1 << 31) | len);
613 } else {
614 *(volatile uint32_t *)(inl + 4 + spc) =
615 SET_BYTE_COUNT((1 << 31) | (len - spc));
616 wmb();
617 *(volatile uint32_t *)inl =
618 SET_BYTE_COUNT((1 << 31) | spc);
619 }
620 }
621
622 static uint32_t hashrandom;
hashrandom_init(void * arg)623 static void hashrandom_init(void *arg)
624 {
625 /*
626 * It is assumed that the random subsystem has been
627 * initialized when this function is called:
628 */
629 hashrandom = m_ether_tcpip_hash_init();
630 }
631 SYSINIT(hashrandom_init, SI_SUB_RANDOM, SI_ORDER_ANY, &hashrandom_init, NULL);
632
mlx4_en_select_queue(struct net_device * dev,struct mbuf * mb)633 u16 mlx4_en_select_queue(struct net_device *dev, struct mbuf *mb)
634 {
635 struct mlx4_en_priv *priv = netdev_priv(dev);
636 u32 rings_p_up = priv->num_tx_rings_p_up;
637 u32 up = 0;
638 u32 queue_index;
639
640 #if (MLX4_EN_NUM_UP > 1)
641 /* Obtain VLAN information if present */
642 if (mb->m_flags & M_VLANTAG) {
643 u32 vlan_tag = mb->m_pkthdr.ether_vtag;
644 up = (vlan_tag >> 13) % MLX4_EN_NUM_UP;
645 }
646 #endif
647 queue_index = m_ether_tcpip_hash(MBUF_HASHFLAG_L3 | MBUF_HASHFLAG_L4, mb, hashrandom);
648
649 return ((queue_index % rings_p_up) + (up * rings_p_up));
650 }
651
mlx4_bf_copy(void __iomem * dst,volatile unsigned long * src,unsigned bytecnt)652 static void mlx4_bf_copy(void __iomem *dst, volatile unsigned long *src, unsigned bytecnt)
653 {
654 __iowrite64_copy(dst, __DEVOLATILE(void *, src), bytecnt / 8);
655 }
656
mlx4_en_mac_to_u64(u8 * addr)657 static u64 mlx4_en_mac_to_u64(u8 *addr)
658 {
659 u64 mac = 0;
660 int i;
661
662 for (i = 0; i < ETHER_ADDR_LEN; i++) {
663 mac <<= 8;
664 mac |= addr[i];
665 }
666 return mac;
667 }
668
mlx4_en_xmit(struct mlx4_en_priv * priv,int tx_ind,struct mbuf ** mbp)669 static int mlx4_en_xmit(struct mlx4_en_priv *priv, int tx_ind, struct mbuf **mbp)
670 {
671 enum {
672 DS_FACT = TXBB_SIZE / DS_SIZE_ALIGNMENT,
673 CTRL_FLAGS = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE |
674 MLX4_WQE_CTRL_SOLICITED),
675 };
676 bus_dma_segment_t segs[MLX4_EN_TX_MAX_MBUF_FRAGS];
677 volatile struct mlx4_wqe_data_seg *dseg;
678 volatile struct mlx4_wqe_data_seg *dseg_inline;
679 volatile struct mlx4_en_tx_desc *tx_desc;
680 struct mlx4_en_tx_ring *ring = priv->tx_ring[tx_ind];
681 struct ifnet *ifp = priv->dev;
682 struct mlx4_en_tx_info *tx_info;
683 struct mbuf *mb = *mbp;
684 struct mbuf *m;
685 __be32 owner_bit;
686 int nr_segs;
687 int pad;
688 int err;
689 u32 bf_size;
690 u32 bf_prod;
691 u32 opcode;
692 u16 index;
693 u16 ds_cnt;
694 u16 ihs;
695
696 if (unlikely(!priv->port_up)) {
697 err = EINVAL;
698 goto tx_drop;
699 }
700
701 /* check if TX ring is full */
702 if (unlikely(mlx4_en_tx_ring_is_full(ring))) {
703 /* every full native Tx ring stops queue */
704 if (ring->blocked == 0)
705 atomic_add_int(&priv->blocked, 1);
706 /* Set HW-queue-is-full flag */
707 atomic_set_int(&ifp->if_drv_flags, IFF_DRV_OACTIVE);
708 priv->port_stats.queue_stopped++;
709 ring->blocked = 1;
710 priv->port_stats.queue_stopped++;
711 ring->queue_stopped++;
712
713 /* Use interrupts to find out when queue opened */
714 mlx4_en_arm_cq(priv, priv->tx_cq[tx_ind]);
715 return (ENOBUFS);
716 }
717
718 /* sanity check we are not wrapping around */
719 KASSERT(((~ring->prod) & ring->size_mask) >=
720 (MLX4_EN_TX_WQE_MAX_WQEBBS - 1), ("Wrapping around TX ring"));
721
722 /* Track current inflight packets for performance analysis */
723 AVG_PERF_COUNTER(priv->pstats.inflight_avg,
724 (u32) (ring->prod - ring->cons - 1));
725
726 /* Track current mbuf packet header length */
727 AVG_PERF_COUNTER(priv->pstats.tx_pktsz_avg, mb->m_pkthdr.len);
728
729 /* Grab an index and try to transmit packet */
730 owner_bit = (ring->prod & ring->size) ?
731 cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0;
732 index = ring->prod & ring->size_mask;
733 tx_desc = (volatile struct mlx4_en_tx_desc *)
734 (ring->buf + index * TXBB_SIZE);
735 tx_info = &ring->tx_info[index];
736 dseg = &tx_desc->data;
737
738 /* send a copy of the frame to the BPF listener, if any */
739 if (ifp != NULL && ifp->if_bpf != NULL)
740 ETHER_BPF_MTAP(ifp, mb);
741
742 /* get default flags */
743 tx_desc->ctrl.srcrb_flags = CTRL_FLAGS;
744
745 if (mb->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO))
746 tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM);
747
748 if (mb->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP |
749 CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO))
750 tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_TCP_UDP_CSUM);
751
752 /* do statistics */
753 if (likely(tx_desc->ctrl.srcrb_flags != CTRL_FLAGS)) {
754 priv->port_stats.tx_chksum_offload++;
755 ring->tx_csum++;
756 }
757
758 /* check for VLAN tag */
759 if (mb->m_flags & M_VLANTAG) {
760 tx_desc->ctrl.vlan_tag = cpu_to_be16(mb->m_pkthdr.ether_vtag);
761 tx_desc->ctrl.ins_vlan = MLX4_WQE_CTRL_INS_VLAN;
762 } else {
763 tx_desc->ctrl.vlan_tag = 0;
764 tx_desc->ctrl.ins_vlan = 0;
765 }
766
767 /* clear immediate field */
768 tx_desc->ctrl.imm = 0;
769
770 /* Handle LSO (TSO) packets */
771 if (mb->m_pkthdr.csum_flags & CSUM_TSO) {
772 u32 payload_len;
773 u32 mss = mb->m_pkthdr.tso_segsz;
774 u32 num_pkts;
775
776 opcode = cpu_to_be32(MLX4_OPCODE_LSO | MLX4_WQE_CTRL_RR) |
777 owner_bit;
778 ihs = mlx4_en_get_header_size(mb);
779 if (unlikely(ihs > MAX_INLINE)) {
780 ring->oversized_packets++;
781 err = EINVAL;
782 goto tx_drop;
783 }
784 tx_desc->lso.mss_hdr_size = cpu_to_be32((mss << 16) | ihs);
785 payload_len = mb->m_pkthdr.len - ihs;
786 if (unlikely(payload_len == 0))
787 num_pkts = 1;
788 else
789 num_pkts = DIV_ROUND_UP(payload_len, mss);
790 ring->bytes += payload_len + (num_pkts * ihs);
791 ring->packets += num_pkts;
792 priv->port_stats.tso_packets++;
793 /* store pointer to inline header */
794 dseg_inline = dseg;
795 /* copy data inline */
796 dseg = mlx4_en_store_inline_lso_data(dseg,
797 mb, ihs, owner_bit);
798 } else {
799 opcode = cpu_to_be32(MLX4_OPCODE_SEND) |
800 owner_bit;
801 ihs = mlx4_en_get_inline_hdr_size(ring, mb);
802 ring->bytes += max_t (unsigned int,
803 mb->m_pkthdr.len, ETHER_MIN_LEN - ETHER_CRC_LEN);
804 ring->packets++;
805 /* store pointer to inline header */
806 dseg_inline = dseg;
807 /* copy data inline */
808 dseg = mlx4_en_store_inline_data(dseg,
809 mb, ihs, owner_bit);
810 }
811 m_adj(mb, ihs);
812
813 /* trim off empty mbufs */
814 while (mb->m_len == 0) {
815 mb = m_free(mb);
816 /* check if all data has been inlined */
817 if (mb == NULL) {
818 nr_segs = 0;
819 goto skip_dma;
820 }
821 }
822
823 err = bus_dmamap_load_mbuf_sg(ring->dma_tag, tx_info->dma_map,
824 mb, segs, &nr_segs, BUS_DMA_NOWAIT);
825 if (unlikely(err == EFBIG)) {
826 /* Too many mbuf fragments */
827 m = m_defrag(mb, M_NOWAIT);
828 if (m == NULL) {
829 ring->oversized_packets++;
830 goto tx_drop;
831 }
832 mb = m;
833 /* Try again */
834 err = bus_dmamap_load_mbuf_sg(ring->dma_tag, tx_info->dma_map,
835 mb, segs, &nr_segs, BUS_DMA_NOWAIT);
836 }
837 /* catch errors */
838 if (unlikely(err != 0)) {
839 ring->oversized_packets++;
840 goto tx_drop;
841 }
842 /* make sure all mbuf data is written to RAM */
843 bus_dmamap_sync(ring->dma_tag, tx_info->dma_map,
844 BUS_DMASYNC_PREWRITE);
845
846 skip_dma:
847 /* compute number of DS needed */
848 ds_cnt = (dseg - ((volatile struct mlx4_wqe_data_seg *)tx_desc)) + nr_segs;
849
850 /*
851 * Check if the next request can wrap around and fill the end
852 * of the current request with zero immediate data:
853 */
854 pad = DIV_ROUND_UP(ds_cnt, DS_FACT);
855 pad = (~(ring->prod + pad)) & ring->size_mask;
856
857 if (unlikely(pad < (MLX4_EN_TX_WQE_MAX_WQEBBS - 1))) {
858 /*
859 * Compute the least number of DS blocks we need to
860 * pad in order to achieve a TX ring wraparound:
861 */
862 pad = (DS_FACT * (pad + 1));
863 } else {
864 /*
865 * The hardware will automatically jump to the next
866 * TXBB. No need for padding.
867 */
868 pad = 0;
869 }
870
871 /* compute total number of DS blocks */
872 ds_cnt += pad;
873 /*
874 * When modifying this code, please ensure that the following
875 * computation is always less than or equal to 0x3F:
876 *
877 * ((MLX4_EN_TX_WQE_MAX_WQEBBS - 1) * DS_FACT) +
878 * (MLX4_EN_TX_WQE_MAX_WQEBBS * DS_FACT)
879 *
880 * Else the "ds_cnt" variable can become too big.
881 */
882 tx_desc->ctrl.fence_size = (ds_cnt & 0x3f);
883
884 /* store pointer to mbuf */
885 tx_info->mb = mb;
886 tx_info->nr_txbb = DIV_ROUND_UP(ds_cnt, DS_FACT);
887 bf_size = ds_cnt * DS_SIZE_ALIGNMENT;
888 bf_prod = ring->prod;
889
890 /* compute end of "dseg" array */
891 dseg += nr_segs + pad;
892
893 /* pad using zero immediate dseg */
894 while (pad--) {
895 dseg--;
896 dseg->addr = 0;
897 dseg->lkey = 0;
898 wmb();
899 dseg->byte_count = SET_BYTE_COUNT((1 << 31)|0);
900 }
901
902 /* fill segment list */
903 while (nr_segs--) {
904 if (unlikely(segs[nr_segs].ds_len == 0)) {
905 dseg--;
906 dseg->addr = 0;
907 dseg->lkey = 0;
908 wmb();
909 dseg->byte_count = SET_BYTE_COUNT((1 << 31)|0);
910 } else {
911 dseg--;
912 dseg->addr = cpu_to_be64((uint64_t)segs[nr_segs].ds_addr);
913 dseg->lkey = cpu_to_be32(priv->mdev->mr.key);
914 wmb();
915 dseg->byte_count = SET_BYTE_COUNT((uint32_t)segs[nr_segs].ds_len);
916 }
917 }
918
919 wmb();
920
921 /* write owner bits in reverse order */
922 if ((opcode & cpu_to_be32(0x1F)) == cpu_to_be32(MLX4_OPCODE_LSO))
923 mlx4_en_store_inline_lso_header(dseg_inline, ihs, owner_bit);
924 else
925 mlx4_en_store_inline_header(dseg_inline, ihs, owner_bit);
926
927 if (unlikely(priv->validate_loopback)) {
928 /* Copy dst mac address to wqe */
929 struct ether_header *ethh;
930 u64 mac;
931 u32 mac_l, mac_h;
932
933 ethh = mtod(mb, struct ether_header *);
934 mac = mlx4_en_mac_to_u64(ethh->ether_dhost);
935 if (mac) {
936 mac_h = (u32) ((mac & 0xffff00000000ULL) >> 16);
937 mac_l = (u32) (mac & 0xffffffff);
938 tx_desc->ctrl.srcrb_flags |= cpu_to_be32(mac_h);
939 tx_desc->ctrl.imm = cpu_to_be32(mac_l);
940 }
941 }
942
943 /* update producer counter */
944 ring->prod += tx_info->nr_txbb;
945
946 if (ring->bf_enabled && bf_size <= MAX_BF &&
947 (tx_desc->ctrl.ins_vlan != MLX4_WQE_CTRL_INS_VLAN)) {
948
949 /* store doorbell number */
950 *(volatile __be32 *) (&tx_desc->ctrl.vlan_tag) |= cpu_to_be32(ring->doorbell_qpn);
951
952 /* or in producer number for this WQE */
953 opcode |= cpu_to_be32((bf_prod & 0xffff) << 8);
954
955 /*
956 * Ensure the new descriptor hits memory before
957 * setting ownership of this descriptor to HW:
958 */
959 wmb();
960 tx_desc->ctrl.owner_opcode = opcode;
961 wmb();
962 mlx4_bf_copy(((u8 *)ring->bf.reg) + ring->bf.offset,
963 (volatile unsigned long *) &tx_desc->ctrl, bf_size);
964 wmb();
965 ring->bf.offset ^= ring->bf.buf_size;
966 } else {
967 /*
968 * Ensure the new descriptor hits memory before
969 * setting ownership of this descriptor to HW:
970 */
971 wmb();
972 tx_desc->ctrl.owner_opcode = opcode;
973 wmb();
974 writel(cpu_to_be32(ring->doorbell_qpn),
975 ((u8 *)ring->bf.uar->map) + MLX4_SEND_DOORBELL);
976 }
977
978 return (0);
979 tx_drop:
980 *mbp = NULL;
981 m_freem(mb);
982 return (err);
983 }
984
985 static int
mlx4_en_transmit_locked(struct ifnet * dev,int tx_ind,struct mbuf * m)986 mlx4_en_transmit_locked(struct ifnet *dev, int tx_ind, struct mbuf *m)
987 {
988 struct mlx4_en_priv *priv = netdev_priv(dev);
989 struct mlx4_en_tx_ring *ring;
990 struct mbuf *next;
991 int enqueued, err = 0;
992
993 ring = priv->tx_ring[tx_ind];
994 if ((dev->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
995 IFF_DRV_RUNNING || priv->port_up == 0) {
996 if (m != NULL)
997 err = drbr_enqueue(dev, ring->br, m);
998 return (err);
999 }
1000
1001 enqueued = 0;
1002 if (m != NULL)
1003 /*
1004 * If we can't insert mbuf into drbr, try to xmit anyway.
1005 * We keep the error we got so we could return that after xmit.
1006 */
1007 err = drbr_enqueue(dev, ring->br, m);
1008
1009 /* Process the queue */
1010 while ((next = drbr_peek(dev, ring->br)) != NULL) {
1011 if (mlx4_en_xmit(priv, tx_ind, &next) != 0) {
1012 if (next == NULL) {
1013 drbr_advance(dev, ring->br);
1014 } else {
1015 drbr_putback(dev, ring->br, next);
1016 }
1017 break;
1018 }
1019 drbr_advance(dev, ring->br);
1020 enqueued++;
1021 if ((dev->if_drv_flags & IFF_DRV_RUNNING) == 0)
1022 break;
1023 }
1024
1025 if (enqueued > 0)
1026 ring->watchdog_time = ticks;
1027
1028 return (err);
1029 }
1030
1031 void
mlx4_en_tx_que(void * context,int pending)1032 mlx4_en_tx_que(void *context, int pending)
1033 {
1034 struct mlx4_en_tx_ring *ring;
1035 struct mlx4_en_priv *priv;
1036 struct net_device *dev;
1037 struct mlx4_en_cq *cq;
1038 int tx_ind;
1039 cq = context;
1040 dev = cq->dev;
1041 priv = dev->if_softc;
1042 tx_ind = cq->ring;
1043 ring = priv->tx_ring[tx_ind];
1044 if (dev->if_drv_flags & IFF_DRV_RUNNING) {
1045 mlx4_en_xmit_poll(priv, tx_ind);
1046 spin_lock(&ring->tx_lock);
1047 if (!drbr_empty(dev, ring->br))
1048 mlx4_en_transmit_locked(dev, tx_ind, NULL);
1049 spin_unlock(&ring->tx_lock);
1050 }
1051 }
1052
1053 int
mlx4_en_transmit(struct ifnet * dev,struct mbuf * m)1054 mlx4_en_transmit(struct ifnet *dev, struct mbuf *m)
1055 {
1056 struct mlx4_en_priv *priv = netdev_priv(dev);
1057 struct mlx4_en_tx_ring *ring;
1058 struct mlx4_en_cq *cq;
1059 int i, err = 0;
1060
1061 /* Compute which queue to use */
1062 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
1063 i = m->m_pkthdr.flowid % priv->tx_ring_num;
1064 }
1065 else {
1066 i = mlx4_en_select_queue(dev, m);
1067 }
1068
1069 ring = priv->tx_ring[i];
1070 if (spin_trylock(&ring->tx_lock)) {
1071 err = mlx4_en_transmit_locked(dev, i, m);
1072 spin_unlock(&ring->tx_lock);
1073 /* Poll CQ here */
1074 mlx4_en_xmit_poll(priv, i);
1075 } else {
1076 err = drbr_enqueue(dev, ring->br, m);
1077 cq = priv->tx_cq[i];
1078 taskqueue_enqueue(cq->tq, &cq->cq_task);
1079 }
1080
1081 return (err);
1082 }
1083
1084 /*
1085 * Flush ring buffers.
1086 */
1087 void
mlx4_en_qflush(struct ifnet * dev)1088 mlx4_en_qflush(struct ifnet *dev)
1089 {
1090 struct mlx4_en_priv *priv = netdev_priv(dev);
1091 struct mlx4_en_tx_ring *ring;
1092 struct mbuf *m;
1093
1094 for (int i = 0; i < priv->tx_ring_num; i++) {
1095 ring = priv->tx_ring[i];
1096 spin_lock(&ring->tx_lock);
1097 while ((m = buf_ring_dequeue_sc(ring->br)) != NULL)
1098 m_freem(m);
1099 spin_unlock(&ring->tx_lock);
1100 }
1101 if_qflush(dev);
1102 }
1103