1 /*
2 * Copyright (c) 2018-2019 Cavium, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
19 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
20 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
21 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
22 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
24 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
25 * POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 /*
29 * File: qlnxr_verbs.c
30 */
31 #include <sys/cdefs.h>
32 #include "qlnxr_def.h"
33 #include "rdma_common.h"
34 #include "qlnxr_roce.h"
35 #include "qlnxr_cm.h"
36
37 #define HILO_U64(hi, lo) ((((u64)(hi)) << 32) + (lo))
38
39 #define TYPEPTR_ADDR_SET(type_ptr, field, vaddr) \
40 do { \
41 (type_ptr)->field.hi = cpu_to_le32(upper_32_bits(vaddr));\
42 (type_ptr)->field.lo = cpu_to_le32(lower_32_bits(vaddr));\
43 } while (0)
44
45 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
46 do { \
47 TYPEPTR_ADDR_SET(sge, addr, vaddr); \
48 (sge)->length = cpu_to_le32(vlength); \
49 (sge)->flags = cpu_to_le32(vflags); \
50 } while (0)
51
52 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
53 do { \
54 TYPEPTR_ADDR_SET(hdr, wr_id, vwr_id); \
55 (hdr)->num_sges = num_sge; \
56 } while (0)
57
58 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
59 do { \
60 TYPEPTR_ADDR_SET(sge, addr, vaddr); \
61 (sge)->length = cpu_to_le32(vlength); \
62 (sge)->l_key = cpu_to_le32(vlkey); \
63 } while (0)
64
65 #define NIPQUAD(addr) \
66 ((unsigned char *)&addr)[0], \
67 ((unsigned char *)&addr)[1], \
68 ((unsigned char *)&addr)[2], \
69 ((unsigned char *)&addr)[3]
70
71 static int
72 qlnxr_check_srq_params(struct ib_pd *ibpd,
73 struct qlnxr_dev *dev,
74 struct ib_srq_init_attr *attrs);
75
76 static int
77 qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx,
78 struct qlnxr_srq *srq,
79 struct qlnxr_create_srq_ureq *ureq,
80 int access, int dmasync);
81
82 static int
83 qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq,
84 struct qlnxr_dev *dev,
85 struct ib_srq_init_attr *init_attr);
86
87 static int
88 qlnxr_copy_srq_uresp(struct qlnxr_dev *dev,
89 struct qlnxr_srq *srq,
90 struct ib_udata *udata);
91
92 static void
93 qlnxr_free_srq_user_params(struct qlnxr_srq *srq);
94
95 static void
96 qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq);
97
98 static u32
99 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq);
100
101 int
qlnxr_iw_query_gid(struct ib_device * ibdev,u8 port,int index,union ib_gid * sgid)102 qlnxr_iw_query_gid(struct ib_device *ibdev, u8 port, int index,
103 union ib_gid *sgid)
104 {
105 struct qlnxr_dev *dev;
106 qlnx_host_t *ha;
107
108 dev = get_qlnxr_dev(ibdev);
109 ha = dev->ha;
110
111 QL_DPRINT12(ha, "enter\n");
112
113 memset(sgid->raw, 0, sizeof(sgid->raw));
114
115 memcpy(sgid->raw, dev->ha->primary_mac, sizeof (dev->ha->primary_mac));
116
117 QL_DPRINT12(ha, "exit\n");
118
119 return 0;
120 }
121
122 int
qlnxr_query_gid(struct ib_device * ibdev,u8 port,int index,union ib_gid * sgid)123 qlnxr_query_gid(struct ib_device *ibdev, u8 port, int index,
124 union ib_gid *sgid)
125 {
126 struct qlnxr_dev *dev;
127 qlnx_host_t *ha;
128
129 dev = get_qlnxr_dev(ibdev);
130 ha = dev->ha;
131 QL_DPRINT12(ha, "enter index: %d\n", index);
132 #if 0
133 int ret = 0;
134 /* @@@: if DEFINE_ROCE_GID_TABLE to be used here */
135 //if (!rdma_cap_roce_gid_table(ibdev, port)) {
136 if (!(rdma_protocol_roce(ibdev, port) &&
137 ibdev->add_gid && ibdev->del_gid)) {
138 QL_DPRINT11(ha, "acquire gid failed\n");
139 return -ENODEV;
140 }
141
142 ret = ib_get_cached_gid(ibdev, port, index, sgid, NULL);
143 if (ret == -EAGAIN) {
144 memcpy(sgid, &zgid, sizeof(*sgid));
145 return 0;
146 }
147 #endif
148 if ((index >= QLNXR_MAX_SGID) || (index < 0)) {
149 QL_DPRINT12(ha, "invalid gid index %d\n", index);
150 memset(sgid, 0, sizeof(*sgid));
151 return -EINVAL;
152 }
153 memcpy(sgid, &dev->sgid_tbl[index], sizeof(*sgid));
154
155 QL_DPRINT12(ha, "exit : %p\n", sgid);
156
157 return 0;
158 }
159
160 struct ib_srq *
qlnxr_create_srq(struct ib_pd * ibpd,struct ib_srq_init_attr * init_attr,struct ib_udata * udata)161 qlnxr_create_srq(struct ib_pd *ibpd, struct ib_srq_init_attr *init_attr,
162 struct ib_udata *udata)
163 {
164 struct qlnxr_dev *dev;
165 qlnx_host_t *ha;
166 struct ecore_rdma_destroy_srq_in_params destroy_in_params;
167 struct ecore_rdma_create_srq_out_params out_params;
168 struct ecore_rdma_create_srq_in_params in_params;
169 u64 pbl_base_addr, phy_prod_pair_addr;
170 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
171 struct ib_ucontext *ib_ctx = NULL;
172 struct qlnxr_srq_hwq_info *hw_srq;
173 struct qlnxr_ucontext *ctx = NULL;
174 struct qlnxr_create_srq_ureq ureq;
175 u32 page_cnt, page_size;
176 struct qlnxr_srq *srq;
177 int ret = 0;
178
179 dev = get_qlnxr_dev((ibpd->device));
180 ha = dev->ha;
181
182 QL_DPRINT12(ha, "enter\n");
183
184 ret = qlnxr_check_srq_params(ibpd, dev, init_attr);
185
186 srq = kzalloc(sizeof(*srq), GFP_KERNEL);
187 if (!srq) {
188 QL_DPRINT11(ha, "cannot allocate memory for srq\n");
189 return NULL; //@@@ : TODO what to return here?
190 }
191
192 srq->dev = dev;
193 hw_srq = &srq->hw_srq;
194 spin_lock_init(&srq->lock);
195 memset(&in_params, 0, sizeof(in_params));
196
197 if (udata && ibpd->uobject && ibpd->uobject->context) {
198 ib_ctx = ibpd->uobject->context;
199 ctx = get_qlnxr_ucontext(ib_ctx);
200
201 memset(&ureq, 0, sizeof(ureq));
202 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
203 udata->inlen))) {
204 QL_DPRINT11(ha, "problem"
205 " copying data from user space\n");
206 goto err0;
207 }
208
209 ret = qlnxr_init_srq_user_params(ib_ctx, srq, &ureq, 0, 0);
210 if (ret)
211 goto err0;
212
213 page_cnt = srq->usrq.pbl_info.num_pbes;
214 pbl_base_addr = srq->usrq.pbl_tbl->pa;
215 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
216 // @@@ : if DEFINE_IB_UMEM_PAGE_SHIFT
217 // page_size = BIT(srq->usrq.umem->page_shift);
218 // else
219 page_size = srq->usrq.umem->page_size;
220 } else {
221 struct ecore_chain *pbl;
222 ret = qlnxr_alloc_srq_kernel_params(srq, dev, init_attr);
223 if (ret)
224 goto err0;
225 pbl = &hw_srq->pbl;
226
227 page_cnt = ecore_chain_get_page_cnt(pbl);
228 pbl_base_addr = ecore_chain_get_pbl_phys(pbl);
229 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
230 page_size = pbl->elem_per_page << 4;
231 }
232
233 in_params.pd_id = pd->pd_id;
234 in_params.pbl_base_addr = pbl_base_addr;
235 in_params.prod_pair_addr = phy_prod_pair_addr;
236 in_params.num_pages = page_cnt;
237 in_params.page_size = page_size;
238
239 ret = ecore_rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
240 if (ret)
241 goto err1;
242
243 srq->srq_id = out_params.srq_id;
244
245 if (udata) {
246 ret = qlnxr_copy_srq_uresp(dev, srq, udata);
247 if (ret)
248 goto err2;
249 }
250
251 QL_DPRINT12(ha, "created srq with srq_id = 0x%0x\n", srq->srq_id);
252 return &srq->ibsrq;
253 err2:
254 memset(&in_params, 0, sizeof(in_params));
255 destroy_in_params.srq_id = srq->srq_id;
256 ecore_rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
257
258 err1:
259 if (udata)
260 qlnxr_free_srq_user_params(srq);
261 else
262 qlnxr_free_srq_kernel_params(srq);
263
264 err0:
265 kfree(srq);
266 return ERR_PTR(-EFAULT);
267 }
268
269 int
qlnxr_destroy_srq(struct ib_srq * ibsrq)270 qlnxr_destroy_srq(struct ib_srq *ibsrq)
271 {
272 struct qlnxr_dev *dev;
273 struct qlnxr_srq *srq;
274 qlnx_host_t *ha;
275 struct ecore_rdma_destroy_srq_in_params in_params;
276
277 srq = get_qlnxr_srq(ibsrq);
278 dev = srq->dev;
279 ha = dev->ha;
280
281 memset(&in_params, 0, sizeof(in_params));
282 in_params.srq_id = srq->srq_id;
283
284 ecore_rdma_destroy_srq(dev->rdma_ctx, &in_params);
285
286 if (ibsrq->pd->uobject && ibsrq->pd->uobject->context)
287 qlnxr_free_srq_user_params(srq);
288 else
289 qlnxr_free_srq_kernel_params(srq);
290
291 QL_DPRINT12(ha, "destroyed srq_id=0x%0x\n", srq->srq_id);
292 kfree(srq);
293 return 0;
294 }
295
296 int
qlnxr_modify_srq(struct ib_srq * ibsrq,struct ib_srq_attr * attr,enum ib_srq_attr_mask attr_mask,struct ib_udata * udata)297 qlnxr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
298 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
299 {
300 struct qlnxr_dev *dev;
301 struct qlnxr_srq *srq;
302 qlnx_host_t *ha;
303 struct ecore_rdma_modify_srq_in_params in_params;
304 int ret = 0;
305
306 srq = get_qlnxr_srq(ibsrq);
307 dev = srq->dev;
308 ha = dev->ha;
309
310 QL_DPRINT12(ha, "enter\n");
311 if (attr_mask & IB_SRQ_MAX_WR) {
312 QL_DPRINT12(ha, "invalid attribute mask=0x%x"
313 " specified for %p\n", attr_mask, srq);
314 return -EINVAL;
315 }
316
317 if (attr_mask & IB_SRQ_LIMIT) {
318 if (attr->srq_limit >= srq->hw_srq.max_wr) {
319 QL_DPRINT12(ha, "invalid srq_limit=0x%x"
320 " (max_srq_limit = 0x%x)\n",
321 attr->srq_limit, srq->hw_srq.max_wr);
322 return -EINVAL;
323 }
324 memset(&in_params, 0, sizeof(in_params));
325 in_params.srq_id = srq->srq_id;
326 in_params.wqe_limit = attr->srq_limit;
327 ret = ecore_rdma_modify_srq(dev->rdma_ctx, &in_params);
328 if (ret)
329 return ret;
330 }
331
332 QL_DPRINT12(ha, "modified srq with srq_id = 0x%0x\n", srq->srq_id);
333 return 0;
334 }
335
336 int
qlnxr_query_srq(struct ib_srq * ibsrq,struct ib_srq_attr * srq_attr)337 qlnxr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
338 {
339 struct qlnxr_dev *dev;
340 struct qlnxr_srq *srq;
341 qlnx_host_t *ha;
342 struct ecore_rdma_device *qattr;
343 srq = get_qlnxr_srq(ibsrq);
344 dev = srq->dev;
345 ha = dev->ha;
346 //qattr = &dev->attr;
347 qattr = ecore_rdma_query_device(dev->rdma_ctx);
348 QL_DPRINT12(ha, "enter\n");
349
350 if (!dev->rdma_ctx) {
351 QL_DPRINT12(ha, "called with invalid params"
352 " rdma_ctx is NULL\n");
353 return -EINVAL;
354 }
355
356 srq_attr->srq_limit = qattr->max_srq;
357 srq_attr->max_wr = qattr->max_srq_wr;
358 srq_attr->max_sge = qattr->max_sge;
359
360 QL_DPRINT12(ha, "exit\n");
361 return 0;
362 }
363
364 /* Increment srq wr producer by one */
365 static
qlnxr_inc_srq_wr_prod(struct qlnxr_srq_hwq_info * info)366 void qlnxr_inc_srq_wr_prod (struct qlnxr_srq_hwq_info *info)
367 {
368 info->wr_prod_cnt++;
369 }
370
371 /* Increment srq wr consumer by one */
372 static
qlnxr_inc_srq_wr_cons(struct qlnxr_srq_hwq_info * info)373 void qlnxr_inc_srq_wr_cons(struct qlnxr_srq_hwq_info *info)
374 {
375 info->wr_cons_cnt++;
376 }
377
378 /* get_port_immutable verb is not available in FreeBSD */
379 #if 0
380 int
qlnxr_roce_port_immutable(struct ib_device * ibdev,u8 port_num,struct ib_port_immutable * immutable)381 qlnxr_roce_port_immutable(struct ib_device *ibdev, u8 port_num,
382 struct ib_port_immutable *immutable)
383 {
384 struct qlnxr_dev *dev;
385 qlnx_host_t *ha;
386 dev = get_qlnxr_dev(ibdev);
387 ha = dev->ha;
388
389 QL_DPRINT12(ha, "entered but not implemented!!!\n");
390 }
391 #endif
392
393 int
qlnxr_post_srq_recv(struct ib_srq * ibsrq,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)394 qlnxr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
395 const struct ib_recv_wr **bad_wr)
396 {
397 struct qlnxr_dev *dev;
398 struct qlnxr_srq *srq;
399 qlnx_host_t *ha;
400 struct qlnxr_srq_hwq_info *hw_srq;
401 struct ecore_chain *pbl;
402 unsigned long flags;
403 int status = 0;
404 u32 num_sge, offset;
405
406 srq = get_qlnxr_srq(ibsrq);
407 dev = srq->dev;
408 ha = dev->ha;
409 hw_srq = &srq->hw_srq;
410
411 QL_DPRINT12(ha, "enter\n");
412 spin_lock_irqsave(&srq->lock, flags);
413
414 pbl = &srq->hw_srq.pbl;
415 while (wr) {
416 struct rdma_srq_wqe_header *hdr;
417 int i;
418
419 if (!qlnxr_srq_elem_left(hw_srq) ||
420 wr->num_sge > srq->hw_srq.max_sges) {
421 QL_DPRINT11(ha, "WR cannot be posted"
422 " (%d, %d) || (%d > %d)\n",
423 hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt,
424 wr->num_sge, srq->hw_srq.max_sges);
425 status = -ENOMEM;
426 *bad_wr = wr;
427 break;
428 }
429
430 hdr = ecore_chain_produce(pbl);
431 num_sge = wr->num_sge;
432 /* Set number of sge and WR id in header */
433 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
434
435 /* PBL is maintained in case of WR granularity.
436 * So increment WR producer in case we post a WR.
437 */
438 qlnxr_inc_srq_wr_prod(hw_srq);
439 hw_srq->wqe_prod++;
440 hw_srq->sge_prod++;
441
442 QL_DPRINT12(ha, "SRQ WR : SGEs: %d with wr_id[%d] = %llx\n",
443 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
444
445 for (i = 0; i < wr->num_sge; i++) {
446 struct rdma_srq_sge *srq_sge =
447 ecore_chain_produce(pbl);
448 /* Set SGE length, lkey and address */
449 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
450 wr->sg_list[i].length, wr->sg_list[i].lkey);
451
452 QL_DPRINT12(ha, "[%d]: len %d, key %x, addr %x:%x\n",
453 i, srq_sge->length, srq_sge->l_key,
454 srq_sge->addr.hi, srq_sge->addr.lo);
455 hw_srq->sge_prod++;
456 }
457 wmb();
458 /*
459 * SRQ prod is 8 bytes. Need to update SGE prod in index
460 * in first 4 bytes and need to update WQE prod in next
461 * 4 bytes.
462 */
463 *(srq->hw_srq.virt_prod_pair_addr) = hw_srq->sge_prod;
464 offset = offsetof(struct rdma_srq_producers, wqe_prod);
465 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) =
466 hw_srq->wqe_prod;
467 /* Flush prod after updating it */
468 wmb();
469 wr = wr->next;
470 }
471
472 QL_DPRINT12(ha, "Elements in SRQ: %d\n",
473 ecore_chain_get_elem_left(pbl));
474
475 spin_unlock_irqrestore(&srq->lock, flags);
476 QL_DPRINT12(ha, "exit\n");
477 return status;
478 }
479
480 int
481 #if __FreeBSD_version < 1102000
qlnxr_query_device(struct ib_device * ibdev,struct ib_device_attr * attr)482 qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr)
483 #else
484 qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
485 struct ib_udata *udata)
486 #endif /* #if __FreeBSD_version < 1102000 */
487
488 {
489 struct qlnxr_dev *dev;
490 struct ecore_rdma_device *qattr;
491 qlnx_host_t *ha;
492
493 dev = get_qlnxr_dev(ibdev);
494 ha = dev->ha;
495
496 QL_DPRINT12(ha, "enter\n");
497
498 #if __FreeBSD_version > 1102000
499 if (udata->inlen || udata->outlen)
500 return -EINVAL;
501 #endif /* #if __FreeBSD_version > 1102000 */
502
503 if (dev->rdma_ctx == NULL) {
504 return -EINVAL;
505 }
506
507 qattr = ecore_rdma_query_device(dev->rdma_ctx);
508
509 memset(attr, 0, sizeof *attr);
510
511 attr->fw_ver = qattr->fw_ver;
512 attr->sys_image_guid = qattr->sys_image_guid;
513 attr->max_mr_size = qattr->max_mr_size;
514 attr->page_size_cap = qattr->page_size_caps;
515 attr->vendor_id = qattr->vendor_id;
516 attr->vendor_part_id = qattr->vendor_part_id;
517 attr->hw_ver = qattr->hw_ver;
518 attr->max_qp = qattr->max_qp;
519 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
520 IB_DEVICE_RC_RNR_NAK_GEN |
521 IB_DEVICE_LOCAL_DMA_LKEY |
522 IB_DEVICE_MEM_MGT_EXTENSIONS;
523
524 attr->max_sge = qattr->max_sge;
525 attr->max_sge_rd = qattr->max_sge;
526 attr->max_cq = qattr->max_cq;
527 attr->max_cqe = qattr->max_cqe;
528 attr->max_mr = qattr->max_mr;
529 attr->max_mw = qattr->max_mw;
530 attr->max_pd = qattr->max_pd;
531 attr->atomic_cap = dev->atomic_cap;
532 attr->max_fmr = qattr->max_fmr;
533 attr->max_map_per_fmr = 16; /* TBD: FMR */
534
535 /* There is an implicit assumption in some of the ib_xxx apps that the
536 * qp_rd_atom is smaller than the qp_init_rd_atom. Specifically, in
537 * communication the qp_rd_atom is passed to the other side and used as
538 * init_rd_atom without check device capabilities for init_rd_atom.
539 * for this reason, we set the qp_rd_atom to be the minimum between the
540 * two...There is an additional assumption in mlx4 driver that the
541 * values are power of two, fls is performed on the value - 1, which
542 * in fact gives a larger power of two for values which are not a power
543 * of two. This should be fixed in mlx4 driver, but until then ->
544 * we provide a value that is a power of two in our code.
545 */
546 attr->max_qp_init_rd_atom =
547 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
548 attr->max_qp_rd_atom =
549 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
550 attr->max_qp_init_rd_atom);
551
552 attr->max_srq = qattr->max_srq;
553 attr->max_srq_sge = qattr->max_srq_sge;
554 attr->max_srq_wr = qattr->max_srq_wr;
555
556 /* TODO: R&D to more properly configure the following */
557 attr->local_ca_ack_delay = qattr->dev_ack_delay;
558 attr->max_fast_reg_page_list_len = qattr->max_mr/8;
559 attr->max_pkeys = QLNXR_ROCE_PKEY_MAX;
560 attr->max_ah = qattr->max_ah;
561
562 QL_DPRINT12(ha, "exit\n");
563 return 0;
564 }
565
566 static inline void
get_link_speed_and_width(int speed,uint8_t * ib_speed,uint8_t * ib_width)567 get_link_speed_and_width(int speed, uint8_t *ib_speed, uint8_t *ib_width)
568 {
569 switch (speed) {
570 case 1000:
571 *ib_speed = IB_SPEED_SDR;
572 *ib_width = IB_WIDTH_1X;
573 break;
574 case 10000:
575 *ib_speed = IB_SPEED_QDR;
576 *ib_width = IB_WIDTH_1X;
577 break;
578
579 case 20000:
580 *ib_speed = IB_SPEED_DDR;
581 *ib_width = IB_WIDTH_4X;
582 break;
583
584 case 25000:
585 *ib_speed = IB_SPEED_EDR;
586 *ib_width = IB_WIDTH_1X;
587 break;
588
589 case 40000:
590 *ib_speed = IB_SPEED_QDR;
591 *ib_width = IB_WIDTH_4X;
592 break;
593
594 case 50000:
595 *ib_speed = IB_SPEED_QDR;
596 *ib_width = IB_WIDTH_4X; // TODO doesn't add up to 50...
597 break;
598
599 case 100000:
600 *ib_speed = IB_SPEED_EDR;
601 *ib_width = IB_WIDTH_4X;
602 break;
603
604 default:
605 /* Unsupported */
606 *ib_speed = IB_SPEED_SDR;
607 *ib_width = IB_WIDTH_1X;
608 }
609 return;
610 }
611
612 int
qlnxr_query_port(struct ib_device * ibdev,uint8_t port,struct ib_port_attr * attr)613 qlnxr_query_port(struct ib_device *ibdev, uint8_t port,
614 struct ib_port_attr *attr)
615 {
616 struct qlnxr_dev *dev;
617 struct ecore_rdma_port *rdma_port;
618 qlnx_host_t *ha;
619
620 dev = get_qlnxr_dev(ibdev);
621 ha = dev->ha;
622
623 QL_DPRINT12(ha, "enter\n");
624
625 if (port > 1) {
626 QL_DPRINT12(ha, "port [%d] > 1 \n", port);
627 return -EINVAL;
628 }
629
630 if (dev->rdma_ctx == NULL) {
631 QL_DPRINT12(ha, "rdma_ctx == NULL\n");
632 return -EINVAL;
633 }
634
635 rdma_port = ecore_rdma_query_port(dev->rdma_ctx);
636 memset(attr, 0, sizeof *attr);
637
638 if (rdma_port->port_state == ECORE_RDMA_PORT_UP) {
639 attr->state = IB_PORT_ACTIVE;
640 attr->phys_state = 5;
641 } else {
642 attr->state = IB_PORT_DOWN;
643 attr->phys_state = 3;
644 }
645
646 attr->max_mtu = IB_MTU_4096;
647 attr->active_mtu = iboe_get_mtu(dev->ha->ifp->if_mtu);
648 attr->lid = 0;
649 attr->lmc = 0;
650 attr->sm_lid = 0;
651 attr->sm_sl = 0;
652 attr->port_cap_flags = 0;
653
654 if (QLNX_IS_IWARP(dev)) {
655 attr->gid_tbl_len = 1;
656 attr->pkey_tbl_len = 1;
657 } else {
658 attr->gid_tbl_len = QLNXR_MAX_SGID;
659 attr->pkey_tbl_len = QLNXR_ROCE_PKEY_TABLE_LEN;
660 }
661
662 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
663 attr->qkey_viol_cntr = 0;
664
665 get_link_speed_and_width(rdma_port->link_speed,
666 &attr->active_speed, &attr->active_width);
667
668 attr->max_msg_sz = rdma_port->max_msg_size;
669 attr->max_vl_num = 4; /* TODO -> figure this one out... */
670
671 QL_DPRINT12(ha, "state = %d phys_state = %d "
672 " link_speed = %d active_speed = %d active_width = %d"
673 " attr->gid_tbl_len = %d attr->pkey_tbl_len = %d"
674 " max_msg_sz = 0x%x max_vl_num = 0x%x \n",
675 attr->state, attr->phys_state,
676 rdma_port->link_speed, attr->active_speed,
677 attr->active_width, attr->gid_tbl_len, attr->pkey_tbl_len,
678 attr->max_msg_sz, attr->max_vl_num);
679
680 QL_DPRINT12(ha, "exit\n");
681 return 0;
682 }
683
684 int
qlnxr_modify_port(struct ib_device * ibdev,uint8_t port,int mask,struct ib_port_modify * props)685 qlnxr_modify_port(struct ib_device *ibdev, uint8_t port, int mask,
686 struct ib_port_modify *props)
687 {
688 struct qlnxr_dev *dev;
689 qlnx_host_t *ha;
690
691 dev = get_qlnxr_dev(ibdev);
692 ha = dev->ha;
693
694 QL_DPRINT12(ha, "enter\n");
695
696 if (port > 1) {
697 QL_DPRINT12(ha, "port (%d) > 1\n", port);
698 return -EINVAL;
699 }
700
701 QL_DPRINT12(ha, "exit\n");
702 return 0;
703 }
704
705 enum rdma_link_layer
qlnxr_link_layer(struct ib_device * ibdev,uint8_t port_num)706 qlnxr_link_layer(struct ib_device *ibdev, uint8_t port_num)
707 {
708 struct qlnxr_dev *dev;
709 qlnx_host_t *ha;
710
711 dev = get_qlnxr_dev(ibdev);
712 ha = dev->ha;
713
714 QL_DPRINT12(ha, "ibdev = %p port_num = 0x%x\n", ibdev, port_num);
715
716 return IB_LINK_LAYER_ETHERNET;
717 }
718
719 struct ib_pd *
qlnxr_alloc_pd(struct ib_device * ibdev,struct ib_ucontext * context,struct ib_udata * udata)720 qlnxr_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context,
721 struct ib_udata *udata)
722 {
723 struct qlnxr_pd *pd = NULL;
724 u16 pd_id;
725 int rc;
726 struct qlnxr_dev *dev;
727 qlnx_host_t *ha;
728
729 dev = get_qlnxr_dev(ibdev);
730 ha = dev->ha;
731
732 QL_DPRINT12(ha, "ibdev = %p context = %p"
733 " udata = %p enter\n", ibdev, context, udata);
734
735 if (dev->rdma_ctx == NULL) {
736 QL_DPRINT11(ha, "dev->rdma_ctx = NULL\n");
737 rc = -1;
738 goto err;
739 }
740
741 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
742 if (!pd) {
743 rc = -ENOMEM;
744 QL_DPRINT11(ha, "kzalloc(pd) = NULL\n");
745 goto err;
746 }
747
748 rc = ecore_rdma_alloc_pd(dev->rdma_ctx, &pd_id);
749 if (rc) {
750 QL_DPRINT11(ha, "ecore_rdma_alloc_pd failed\n");
751 goto err;
752 }
753
754 pd->pd_id = pd_id;
755
756 if (udata && context) {
757 rc = ib_copy_to_udata(udata, &pd->pd_id, sizeof(pd->pd_id));
758 if (rc) {
759 QL_DPRINT11(ha, "ib_copy_to_udata failed\n");
760 ecore_rdma_free_pd(dev->rdma_ctx, pd_id);
761 goto err;
762 }
763
764 pd->uctx = get_qlnxr_ucontext(context);
765 pd->uctx->pd = pd;
766 }
767
768 atomic_add_rel_32(&dev->pd_count, 1);
769 QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n",
770 pd, pd_id, dev->pd_count);
771
772 return &pd->ibpd;
773
774 err:
775 kfree(pd);
776 QL_DPRINT12(ha, "exit -1\n");
777 return ERR_PTR(rc);
778 }
779
780 int
qlnxr_dealloc_pd(struct ib_pd * ibpd)781 qlnxr_dealloc_pd(struct ib_pd *ibpd)
782 {
783 struct qlnxr_pd *pd;
784 struct qlnxr_dev *dev;
785 qlnx_host_t *ha;
786
787 pd = get_qlnxr_pd(ibpd);
788 dev = get_qlnxr_dev((ibpd->device));
789 ha = dev->ha;
790
791 QL_DPRINT12(ha, "enter\n");
792
793 if (pd == NULL) {
794 QL_DPRINT11(ha, "pd = NULL\n");
795 } else {
796 ecore_rdma_free_pd(dev->rdma_ctx, pd->pd_id);
797 kfree(pd);
798 atomic_subtract_rel_32(&dev->pd_count, 1);
799 QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n",
800 pd, pd->pd_id, dev->pd_count);
801 }
802
803 QL_DPRINT12(ha, "exit\n");
804 return 0;
805 }
806
807 #define ROCE_WQE_ELEM_SIZE sizeof(struct rdma_sq_sge)
808 #define RDMA_MAX_SGE_PER_SRQ (4) /* Should be part of HSI */
809 /* Should be part of HSI */
810 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1) /* +1 for header */
811 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
812
813 static void qlnxr_cleanup_user(struct qlnxr_dev *, struct qlnxr_qp *);
814 static void qlnxr_cleanup_kernel(struct qlnxr_dev *, struct qlnxr_qp *);
815
816 int
qlnxr_query_pkey(struct ib_device * ibdev,u8 port,u16 index,u16 * pkey)817 qlnxr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
818 {
819 struct qlnxr_dev *dev;
820 qlnx_host_t *ha;
821
822 dev = get_qlnxr_dev(ibdev);
823 ha = dev->ha;
824
825 QL_DPRINT12(ha, "enter index = 0x%x\n", index);
826
827 if (index > QLNXR_ROCE_PKEY_TABLE_LEN)
828 return -EINVAL;
829
830 *pkey = QLNXR_ROCE_PKEY_DEFAULT;
831
832 QL_DPRINT12(ha, "exit\n");
833 return 0;
834 }
835
836 static inline bool
qlnxr_get_vlan_id_qp(qlnx_host_t * ha,struct ib_qp_attr * attr,int attr_mask,u16 * vlan_id)837 qlnxr_get_vlan_id_qp(qlnx_host_t *ha, struct ib_qp_attr *attr, int attr_mask,
838 u16 *vlan_id)
839 {
840 bool ret = false;
841
842 QL_DPRINT12(ha, "enter \n");
843
844 *vlan_id = 0;
845
846 #if __FreeBSD_version >= 1100000
847 u16 tmp_vlan_id;
848
849 #if __FreeBSD_version >= 1102000
850 union ib_gid *dgid;
851
852 dgid = &attr->ah_attr.grh.dgid;
853 tmp_vlan_id = (dgid->raw[11] << 8) | dgid->raw[12];
854
855 if (!(tmp_vlan_id & ~EVL_VLID_MASK)) {
856 *vlan_id = tmp_vlan_id;
857 ret = true;
858 }
859 #else
860 tmp_vlan_id = attr->vlan_id;
861
862 if ((attr_mask & IB_QP_VID) && (!(tmp_vlan_id & ~EVL_VLID_MASK))) {
863 *vlan_id = tmp_vlan_id;
864 ret = true;
865 }
866
867 #endif /* #if __FreeBSD_version > 1102000 */
868
869 #else
870 ret = true;
871
872 #endif /* #if __FreeBSD_version >= 1100000 */
873
874 QL_DPRINT12(ha, "exit vlan_id = 0x%x ret = %d \n", *vlan_id, ret);
875
876 return (ret);
877 }
878
879 static inline void
get_gid_info(struct ib_qp * ibqp,struct ib_qp_attr * attr,int attr_mask,struct qlnxr_dev * dev,struct qlnxr_qp * qp,struct ecore_rdma_modify_qp_in_params * qp_params)880 get_gid_info(struct ib_qp *ibqp, struct ib_qp_attr *attr,
881 int attr_mask,
882 struct qlnxr_dev *dev,
883 struct qlnxr_qp *qp,
884 struct ecore_rdma_modify_qp_in_params *qp_params)
885 {
886 int i;
887 qlnx_host_t *ha;
888
889 ha = dev->ha;
890
891 QL_DPRINT12(ha, "enter\n");
892
893 memcpy(&qp_params->sgid.bytes[0],
894 &dev->sgid_tbl[qp->sgid_idx].raw[0],
895 sizeof(qp_params->sgid.bytes));
896 memcpy(&qp_params->dgid.bytes[0],
897 &attr->ah_attr.grh.dgid.raw[0],
898 sizeof(qp_params->dgid));
899
900 qlnxr_get_vlan_id_qp(ha, attr, attr_mask, &qp_params->vlan_id);
901
902 for (i = 0; i < (sizeof(qp_params->sgid.dwords)/sizeof(uint32_t)); i++) {
903 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
904 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
905 }
906
907 QL_DPRINT12(ha, "exit\n");
908 return;
909 }
910
911 static int
qlnxr_add_mmap(struct qlnxr_ucontext * uctx,u64 phy_addr,unsigned long len)912 qlnxr_add_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len)
913 {
914 struct qlnxr_mm *mm;
915 qlnx_host_t *ha;
916
917 ha = uctx->dev->ha;
918
919 QL_DPRINT12(ha, "enter\n");
920
921 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
922 if (mm == NULL) {
923 QL_DPRINT11(ha, "mm = NULL\n");
924 return -ENOMEM;
925 }
926
927 mm->key.phy_addr = phy_addr;
928
929 /* This function might be called with a length which is not a multiple
930 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
931 * forces this granularity by increasing the requested size if needed.
932 * When qedr_mmap is called, it will search the list with the updated
933 * length as a key. To prevent search failures, the length is rounded up
934 * in advance to PAGE_SIZE.
935 */
936 mm->key.len = roundup(len, PAGE_SIZE);
937 INIT_LIST_HEAD(&mm->entry);
938
939 mutex_lock(&uctx->mm_list_lock);
940 list_add(&mm->entry, &uctx->mm_head);
941 mutex_unlock(&uctx->mm_list_lock);
942
943 QL_DPRINT12(ha, "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
944 (unsigned long long)mm->key.phy_addr,
945 (unsigned long)mm->key.len, uctx);
946
947 return 0;
948 }
949
950 static bool
qlnxr_search_mmap(struct qlnxr_ucontext * uctx,u64 phy_addr,unsigned long len)951 qlnxr_search_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len)
952 {
953 bool found = false;
954 struct qlnxr_mm *mm;
955 qlnx_host_t *ha;
956
957 ha = uctx->dev->ha;
958
959 QL_DPRINT12(ha, "enter\n");
960
961 mutex_lock(&uctx->mm_list_lock);
962 list_for_each_entry(mm, &uctx->mm_head, entry) {
963 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
964 continue;
965
966 found = true;
967 break;
968 }
969 mutex_unlock(&uctx->mm_list_lock);
970
971 QL_DPRINT12(ha,
972 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, found=%d\n",
973 mm->key.phy_addr, mm->key.len, uctx, found);
974
975 return found;
976 }
977
978 struct
qlnxr_alloc_ucontext(struct ib_device * ibdev,struct ib_udata * udata)979 ib_ucontext *qlnxr_alloc_ucontext(struct ib_device *ibdev,
980 struct ib_udata *udata)
981 {
982 int rc;
983 struct qlnxr_ucontext *ctx;
984 struct qlnxr_alloc_ucontext_resp uresp;
985 struct qlnxr_dev *dev = get_qlnxr_dev(ibdev);
986 qlnx_host_t *ha = dev->ha;
987 struct ecore_rdma_add_user_out_params oparams;
988
989 if (!udata) {
990 return ERR_PTR(-EFAULT);
991 }
992
993 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
994 if (!ctx)
995 return ERR_PTR(-ENOMEM);
996
997 rc = ecore_rdma_add_user(dev->rdma_ctx, &oparams);
998 if (rc) {
999 QL_DPRINT12(ha,
1000 "Failed to allocate a DPI for a new RoCE application "
1001 ",rc = %d. To overcome this, consider to increase "
1002 "the number of DPIs, increase the doorbell BAR size "
1003 "or just close unnecessary RoCE applications. In "
1004 "order to increase the number of DPIs consult the "
1005 "README\n", rc);
1006 goto err;
1007 }
1008
1009 ctx->dpi = oparams.dpi;
1010 ctx->dpi_addr = oparams.dpi_addr;
1011 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
1012 ctx->dpi_size = oparams.dpi_size;
1013 INIT_LIST_HEAD(&ctx->mm_head);
1014 mutex_init(&ctx->mm_list_lock);
1015
1016 memset(&uresp, 0, sizeof(uresp));
1017 uresp.dpm_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, dpm_enabled)
1018 < udata->outlen ? dev->user_dpm_enabled : 0; //TODO: figure this out
1019 uresp.wids_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, wids_enabled)
1020 < udata->outlen ? 1 : 0; //TODO: figure this out
1021 uresp.wid_count = offsetof(struct qlnxr_alloc_ucontext_resp, wid_count)
1022 < udata->outlen ? oparams.wid_count : 0; //TODO: figure this out
1023 uresp.db_pa = ctx->dpi_phys_addr;
1024 uresp.db_size = ctx->dpi_size;
1025 uresp.max_send_wr = dev->attr.max_sqe;
1026 uresp.max_recv_wr = dev->attr.max_rqe;
1027 uresp.max_srq_wr = dev->attr.max_srq_wr;
1028 uresp.sges_per_send_wr = QLNXR_MAX_SQE_ELEMENTS_PER_SQE;
1029 uresp.sges_per_recv_wr = QLNXR_MAX_RQE_ELEMENTS_PER_RQE;
1030 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
1031 uresp.max_cqes = QLNXR_MAX_CQES;
1032
1033 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1034 if (rc)
1035 goto err;
1036
1037 ctx->dev = dev;
1038
1039 rc = qlnxr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
1040 if (rc)
1041 goto err;
1042 QL_DPRINT12(ha, "Allocated user context %p\n",
1043 &ctx->ibucontext);
1044
1045 return &ctx->ibucontext;
1046 err:
1047 kfree(ctx);
1048 return ERR_PTR(rc);
1049 }
1050
1051 int
qlnxr_dealloc_ucontext(struct ib_ucontext * ibctx)1052 qlnxr_dealloc_ucontext(struct ib_ucontext *ibctx)
1053 {
1054 struct qlnxr_ucontext *uctx = get_qlnxr_ucontext(ibctx);
1055 struct qlnxr_dev *dev = uctx->dev;
1056 qlnx_host_t *ha = dev->ha;
1057 struct qlnxr_mm *mm, *tmp;
1058 int status = 0;
1059
1060 QL_DPRINT12(ha, "Deallocating user context %p\n",
1061 uctx);
1062
1063 if (dev) {
1064 ecore_rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
1065 }
1066
1067 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
1068 QL_DPRINT12(ha, "deleted addr= 0x%llx, len = 0x%lx for"
1069 " ctx=%p\n",
1070 mm->key.phy_addr, mm->key.len, uctx);
1071 list_del(&mm->entry);
1072 kfree(mm);
1073 }
1074 kfree(uctx);
1075 return status;
1076 }
1077
1078 int
qlnxr_mmap(struct ib_ucontext * context,struct vm_area_struct * vma)1079 qlnxr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
1080 {
1081 struct qlnxr_ucontext *ucontext = get_qlnxr_ucontext(context);
1082 struct qlnxr_dev *dev = get_qlnxr_dev((context->device));
1083 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
1084 u64 unmapped_db;
1085 unsigned long len = (vma->vm_end - vma->vm_start);
1086 int rc = 0;
1087 bool found;
1088 qlnx_host_t *ha;
1089
1090 ha = dev->ha;
1091
1092 #if __FreeBSD_version > 1102000
1093 unmapped_db = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size);
1094 #else
1095 unmapped_db = dev->db_phys_addr;
1096 #endif /* #if __FreeBSD_version > 1102000 */
1097
1098 QL_DPRINT12(ha, "qedr_mmap enter vm_page=0x%lx"
1099 " vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n",
1100 vm_page, vma->vm_pgoff, unmapped_db,
1101 dev->db_size, len);
1102
1103 if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) {
1104 QL_DPRINT11(ha, "Vma_start not page aligned "
1105 "vm_start = %ld vma_end = %ld\n", vma->vm_start,
1106 vma->vm_end);
1107 return -EINVAL;
1108 }
1109
1110 found = qlnxr_search_mmap(ucontext, vm_page, len);
1111 if (!found) {
1112 QL_DPRINT11(ha, "Vma_pgoff not found in mapped array = %ld\n",
1113 vma->vm_pgoff);
1114 return -EINVAL;
1115 }
1116
1117 QL_DPRINT12(ha, "Mapping doorbell bar\n");
1118
1119 #if __FreeBSD_version > 1102000
1120
1121 if ((vm_page < unmapped_db) ||
1122 ((vm_page + len) > (unmapped_db + ucontext->dpi_size))) {
1123 QL_DPRINT11(ha, "failed pages are outside of dpi;"
1124 "page address=0x%lx, unmapped_db=0x%lx, dpi_size=0x%x\n",
1125 vm_page, unmapped_db, ucontext->dpi_size);
1126 return -EINVAL;
1127 }
1128
1129 if (vma->vm_flags & VM_READ) {
1130 QL_DPRINT11(ha, "failed mmap, cannot map doorbell bar for read\n");
1131 return -EINVAL;
1132 }
1133
1134 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
1135 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len,
1136 vma->vm_page_prot);
1137
1138 #else
1139
1140 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
1141 dev->db_size))) {
1142 QL_DPRINT12(ha, "Mapping doorbell bar\n");
1143
1144 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
1145
1146 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
1147 PAGE_SIZE, vma->vm_page_prot);
1148 } else {
1149 QL_DPRINT12(ha, "Mapping chains\n");
1150 rc = io_remap_pfn_range(vma, vma->vm_start,
1151 vma->vm_pgoff, len, vma->vm_page_prot);
1152 }
1153
1154 #endif /* #if __FreeBSD_version > 1102000 */
1155
1156 QL_DPRINT12(ha, "exit [%d]\n", rc);
1157 return rc;
1158 }
1159
1160 struct ib_mr *
qlnxr_get_dma_mr(struct ib_pd * ibpd,int acc)1161 qlnxr_get_dma_mr(struct ib_pd *ibpd, int acc)
1162 {
1163 struct qlnxr_mr *mr;
1164 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
1165 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
1166 int rc;
1167 qlnx_host_t *ha;
1168
1169 ha = dev->ha;
1170
1171 QL_DPRINT12(ha, "enter\n");
1172
1173 if (acc & IB_ACCESS_MW_BIND) {
1174 QL_DPRINT12(ha, "Unsupported access flags received for dma mr\n");
1175 }
1176
1177 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1178 if (!mr) {
1179 rc = -ENOMEM;
1180 QL_DPRINT12(ha, "kzalloc(mr) failed %d\n", rc);
1181 goto err0;
1182 }
1183
1184 mr->type = QLNXR_MR_DMA;
1185
1186 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
1187 if (rc) {
1188 QL_DPRINT12(ha, "ecore_rdma_alloc_tid failed %d\n", rc);
1189 goto err1;
1190 }
1191
1192 /* index only, 18 bit long, lkey = itid << 8 | key */
1193 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR;
1194 mr->hw_mr.pd = pd->pd_id;
1195 mr->hw_mr.local_read = 1;
1196 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
1197 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
1198 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
1199 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
1200 mr->hw_mr.dma_mr = true;
1201
1202 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
1203 if (rc) {
1204 QL_DPRINT12(ha, "ecore_rdma_register_tid failed %d\n", rc);
1205 goto err2;
1206 }
1207
1208 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
1209
1210 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
1211 mr->hw_mr.remote_atomic) {
1212 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
1213 }
1214
1215 QL_DPRINT12(ha, "lkey = %x\n", mr->ibmr.lkey);
1216
1217 return &mr->ibmr;
1218
1219 err2:
1220 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
1221 err1:
1222 kfree(mr);
1223 err0:
1224 QL_DPRINT12(ha, "exit [%d]\n", rc);
1225
1226 return ERR_PTR(rc);
1227 }
1228
1229 static void
qlnxr_free_pbl(struct qlnxr_dev * dev,struct qlnxr_pbl_info * pbl_info,struct qlnxr_pbl * pbl)1230 qlnxr_free_pbl(struct qlnxr_dev *dev, struct qlnxr_pbl_info *pbl_info,
1231 struct qlnxr_pbl *pbl)
1232 {
1233 int i;
1234 qlnx_host_t *ha;
1235
1236 ha = dev->ha;
1237
1238 QL_DPRINT12(ha, "enter\n");
1239
1240 for (i = 0; i < pbl_info->num_pbls; i++) {
1241 if (!pbl[i].va)
1242 continue;
1243 qlnx_dma_free_coherent(&dev->ha->cdev, pbl[i].va, pbl[i].pa,
1244 pbl_info->pbl_size);
1245 }
1246 kfree(pbl);
1247
1248 QL_DPRINT12(ha, "exit\n");
1249 return;
1250 }
1251
1252 #define MIN_FW_PBL_PAGE_SIZE (4*1024)
1253 #define MAX_FW_PBL_PAGE_SIZE (64*1024)
1254
1255 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
1256 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
1257 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE*MAX_PBES_ON_PAGE)
1258
1259 static struct qlnxr_pbl *
qlnxr_alloc_pbl_tbl(struct qlnxr_dev * dev,struct qlnxr_pbl_info * pbl_info,gfp_t flags)1260 qlnxr_alloc_pbl_tbl(struct qlnxr_dev *dev,
1261 struct qlnxr_pbl_info *pbl_info, gfp_t flags)
1262 {
1263 void *va;
1264 dma_addr_t pa;
1265 dma_addr_t *pbl_main_tbl;
1266 struct qlnxr_pbl *pbl_table;
1267 int i, rc = 0;
1268 qlnx_host_t *ha;
1269
1270 ha = dev->ha;
1271
1272 QL_DPRINT12(ha, "enter\n");
1273
1274 pbl_table = kzalloc(sizeof(*pbl_table) * pbl_info->num_pbls, flags);
1275
1276 if (!pbl_table) {
1277 QL_DPRINT12(ha, "pbl_table = NULL\n");
1278 return NULL;
1279 }
1280
1281 for (i = 0; i < pbl_info->num_pbls; i++) {
1282 va = qlnx_dma_alloc_coherent(&dev->ha->cdev, &pa, pbl_info->pbl_size);
1283 if (!va) {
1284 QL_DPRINT11(ha, "Failed to allocate pbl#%d\n", i);
1285 rc = -ENOMEM;
1286 goto err;
1287 }
1288 memset(va, 0, pbl_info->pbl_size);
1289 pbl_table[i].va = va;
1290 pbl_table[i].pa = pa;
1291 }
1292
1293 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
1294 * the first one with physical pointers to all of the rest
1295 */
1296 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
1297 for (i = 0; i < pbl_info->num_pbls - 1; i++)
1298 pbl_main_tbl[i] = pbl_table[i + 1].pa;
1299
1300 QL_DPRINT12(ha, "exit\n");
1301 return pbl_table;
1302
1303 err:
1304 qlnxr_free_pbl(dev, pbl_info, pbl_table);
1305
1306 QL_DPRINT12(ha, "exit with error\n");
1307 return NULL;
1308 }
1309
1310 static int
qlnxr_prepare_pbl_tbl(struct qlnxr_dev * dev,struct qlnxr_pbl_info * pbl_info,u32 num_pbes,int two_layer_capable)1311 qlnxr_prepare_pbl_tbl(struct qlnxr_dev *dev,
1312 struct qlnxr_pbl_info *pbl_info,
1313 u32 num_pbes,
1314 int two_layer_capable)
1315 {
1316 u32 pbl_capacity;
1317 u32 pbl_size;
1318 u32 num_pbls;
1319 qlnx_host_t *ha;
1320
1321 ha = dev->ha;
1322
1323 QL_DPRINT12(ha, "enter\n");
1324
1325 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
1326 if (num_pbes > MAX_PBES_TWO_LAYER) {
1327 QL_DPRINT11(ha, "prepare pbl table: too many pages %d\n",
1328 num_pbes);
1329 return -EINVAL;
1330 }
1331
1332 /* calculate required pbl page size */
1333 pbl_size = MIN_FW_PBL_PAGE_SIZE;
1334 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
1335 NUM_PBES_ON_PAGE(pbl_size);
1336
1337 while (pbl_capacity < num_pbes) {
1338 pbl_size *= 2;
1339 pbl_capacity = pbl_size / sizeof(u64);
1340 pbl_capacity = pbl_capacity * pbl_capacity;
1341 }
1342
1343 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
1344 num_pbls++; /* One for the layer0 ( points to the pbls) */
1345 pbl_info->two_layered = true;
1346 } else {
1347 /* One layered PBL */
1348 num_pbls = 1;
1349 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, \
1350 roundup_pow_of_two((num_pbes * sizeof(u64))));
1351 pbl_info->two_layered = false;
1352 }
1353
1354 pbl_info->num_pbls = num_pbls;
1355 pbl_info->pbl_size = pbl_size;
1356 pbl_info->num_pbes = num_pbes;
1357
1358 QL_DPRINT12(ha, "prepare pbl table: num_pbes=%d, num_pbls=%d pbl_size=%d\n",
1359 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
1360
1361 return 0;
1362 }
1363
1364 static void
qlnxr_populate_pbls(struct qlnxr_dev * dev,struct ib_umem * umem,struct qlnxr_pbl * pbl,struct qlnxr_pbl_info * pbl_info)1365 qlnxr_populate_pbls(struct qlnxr_dev *dev, struct ib_umem *umem,
1366 struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info)
1367 {
1368 struct regpair *pbe;
1369 struct qlnxr_pbl *pbl_tbl;
1370 struct scatterlist *sg;
1371 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
1372 qlnx_host_t *ha;
1373
1374 #ifdef DEFINE_IB_UMEM_WITH_CHUNK
1375 int i;
1376 struct ib_umem_chunk *chunk = NULL;
1377 #else
1378 int entry;
1379 #endif
1380
1381 ha = dev->ha;
1382
1383 QL_DPRINT12(ha, "enter\n");
1384
1385 if (!pbl_info) {
1386 QL_DPRINT11(ha, "PBL_INFO not initialized\n");
1387 return;
1388 }
1389
1390 if (!pbl_info->num_pbes) {
1391 QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n");
1392 return;
1393 }
1394
1395 /* If we have a two layered pbl, the first pbl points to the rest
1396 * of the pbls and the first entry lays on the second pbl in the table
1397 */
1398 if (pbl_info->two_layered)
1399 pbl_tbl = &pbl[1];
1400 else
1401 pbl_tbl = pbl;
1402
1403 pbe = (struct regpair *)pbl_tbl->va;
1404 if (!pbe) {
1405 QL_DPRINT12(ha, "pbe is NULL\n");
1406 return;
1407 }
1408
1409 pbe_cnt = 0;
1410
1411 shift = ilog2(umem->page_size);
1412
1413 #ifndef DEFINE_IB_UMEM_WITH_CHUNK
1414
1415 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
1416 #else
1417 list_for_each_entry(chunk, &umem->chunk_list, list) {
1418 /* get all the dma regions from the chunk. */
1419 for (i = 0; i < chunk->nmap; i++) {
1420 sg = &chunk->page_list[i];
1421 #endif
1422 pages = sg_dma_len(sg) >> shift;
1423 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
1424 /* store the page address in pbe */
1425 pbe->lo =
1426 cpu_to_le32(sg_dma_address(sg) +
1427 (umem->page_size * pg_cnt));
1428 pbe->hi =
1429 cpu_to_le32(upper_32_bits
1430 ((sg_dma_address(sg) +
1431 umem->page_size * pg_cnt)));
1432
1433 QL_DPRINT12(ha,
1434 "Populate pbl table:"
1435 " pbe->addr=0x%x:0x%x "
1436 " pbe_cnt = %d total_num_pbes=%d"
1437 " pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt,
1438 total_num_pbes, pbe);
1439
1440 pbe_cnt ++;
1441 total_num_pbes ++;
1442 pbe++;
1443
1444 if (total_num_pbes == pbl_info->num_pbes)
1445 return;
1446
1447 /* if the given pbl is full storing the pbes,
1448 * move to next pbl.
1449 */
1450 if (pbe_cnt ==
1451 (pbl_info->pbl_size / sizeof(u64))) {
1452 pbl_tbl++;
1453 pbe = (struct regpair *)pbl_tbl->va;
1454 pbe_cnt = 0;
1455 }
1456 }
1457 #ifdef DEFINE_IB_UMEM_WITH_CHUNK
1458 }
1459 #endif
1460 }
1461 QL_DPRINT12(ha, "exit\n");
1462 return;
1463 }
1464
1465 static void
1466 free_mr_info(struct qlnxr_dev *dev, struct mr_info *info)
1467 {
1468 struct qlnxr_pbl *pbl, *tmp;
1469 qlnx_host_t *ha;
1470
1471 ha = dev->ha;
1472
1473 QL_DPRINT12(ha, "enter\n");
1474
1475 if (info->pbl_table)
1476 list_add_tail(&info->pbl_table->list_entry,
1477 &info->free_pbl_list);
1478
1479 if (!list_empty(&info->inuse_pbl_list))
1480 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
1481
1482 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
1483 list_del(&pbl->list_entry);
1484 qlnxr_free_pbl(dev, &info->pbl_info, pbl);
1485 }
1486 QL_DPRINT12(ha, "exit\n");
1487
1488 return;
1489 }
1490
1491 static int
1492 qlnxr_init_mr_info(struct qlnxr_dev *dev, struct mr_info *info,
1493 size_t page_list_len, bool two_layered)
1494 {
1495 int rc;
1496 struct qlnxr_pbl *tmp;
1497 qlnx_host_t *ha;
1498
1499 ha = dev->ha;
1500
1501 QL_DPRINT12(ha, "enter\n");
1502
1503 INIT_LIST_HEAD(&info->free_pbl_list);
1504 INIT_LIST_HEAD(&info->inuse_pbl_list);
1505
1506 rc = qlnxr_prepare_pbl_tbl(dev, &info->pbl_info,
1507 page_list_len, two_layered);
1508 if (rc) {
1509 QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl [%d]\n", rc);
1510 goto done;
1511 }
1512
1513 info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
1514
1515 if (!info->pbl_table) {
1516 rc = -ENOMEM;
1517 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl returned NULL\n");
1518 goto done;
1519 }
1520
1521 QL_DPRINT12(ha, "pbl_table_pa = %pa\n", &info->pbl_table->pa);
1522
1523 /* in usual case we use 2 PBLs, so we add one to free
1524 * list and allocating another one
1525 */
1526 tmp = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
1527
1528 if (!tmp) {
1529 QL_DPRINT11(ha, "Extra PBL is not allocated\n");
1530 goto done; /* it's OK if second allocation fails, so rc = 0*/
1531 }
1532
1533 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
1534
1535 QL_DPRINT12(ha, "extra pbl_table_pa = %pa\n", &tmp->pa);
1536
1537 done:
1538 if (rc)
1539 free_mr_info(dev, info);
1540
1541 QL_DPRINT12(ha, "exit [%d]\n", rc);
1542
1543 return rc;
1544 }
1545
1546 struct ib_mr *
1547 #if __FreeBSD_version >= 1102000
1548 qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
1549 u64 usr_addr, int acc, struct ib_udata *udata)
1550 #else
1551 qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
1552 u64 usr_addr, int acc, struct ib_udata *udata, int mr_id)
1553 #endif /* #if __FreeBSD_version >= 1102000 */
1554 {
1555 int rc = -ENOMEM;
1556 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
1557 struct qlnxr_mr *mr;
1558 struct qlnxr_pd *pd;
1559 qlnx_host_t *ha;
1560
1561 ha = dev->ha;
1562
1563 QL_DPRINT12(ha, "enter\n");
1564
1565 pd = get_qlnxr_pd(ibpd);
1566
1567 QL_DPRINT12(ha, "qedr_register user mr pd = %d"
1568 " start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
1569 pd->pd_id, start, len, usr_addr, acc);
1570
1571 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {
1572 QL_DPRINT11(ha,
1573 "(acc & IB_ACCESS_REMOTE_WRITE &&"
1574 " !(acc & IB_ACCESS_LOCAL_WRITE))\n");
1575 return ERR_PTR(-EINVAL);
1576 }
1577
1578 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1579 if (!mr) {
1580 QL_DPRINT11(ha, "kzalloc(mr) failed\n");
1581 return ERR_PTR(rc);
1582 }
1583
1584 mr->type = QLNXR_MR_USER;
1585
1586 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
1587 if (IS_ERR(mr->umem)) {
1588 rc = -EFAULT;
1589 QL_DPRINT11(ha, "ib_umem_get failed [%p]\n", mr->umem);
1590 goto err0;
1591 }
1592
1593 rc = qlnxr_init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
1594 if (rc) {
1595 QL_DPRINT11(ha,
1596 "qlnxr_init_mr_info failed [%d]\n", rc);
1597 goto err1;
1598 }
1599
1600 qlnxr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
1601 &mr->info.pbl_info);
1602
1603 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
1604
1605 if (rc) {
1606 QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc);
1607 goto err1;
1608 }
1609
1610 /* index only, 18 bit long, lkey = itid << 8 | key */
1611 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR;
1612 mr->hw_mr.key = 0;
1613 mr->hw_mr.pd = pd->pd_id;
1614 mr->hw_mr.local_read = 1;
1615 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
1616 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
1617 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
1618 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
1619 mr->hw_mr.mw_bind = false; /* TBD MW BIND */
1620 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
1621 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
1622 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
1623 mr->hw_mr.page_size_log = ilog2(mr->umem->page_size); /* for the MR pages */
1624
1625 #if __FreeBSD_version >= 1102000
1626 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
1627 #else
1628 mr->hw_mr.fbo = mr->umem->offset;
1629 #endif
1630 mr->hw_mr.length = len;
1631 mr->hw_mr.vaddr = usr_addr;
1632 mr->hw_mr.zbva = false; /* TBD figure when this should be true */
1633 mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */
1634 mr->hw_mr.dma_mr = false;
1635
1636 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
1637 if (rc) {
1638 QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc);
1639 goto err2;
1640 }
1641
1642 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
1643 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
1644 mr->hw_mr.remote_atomic)
1645 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
1646
1647 QL_DPRINT12(ha, "register user mr lkey: %x\n", mr->ibmr.lkey);
1648
1649 return (&mr->ibmr);
1650
1651 err2:
1652 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
1653 err1:
1654 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
1655 err0:
1656 kfree(mr);
1657
1658 QL_DPRINT12(ha, "exit [%d]\n", rc);
1659 return (ERR_PTR(rc));
1660 }
1661
1662 int
1663 qlnxr_dereg_mr(struct ib_mr *ib_mr)
1664 {
1665 struct qlnxr_mr *mr = get_qlnxr_mr(ib_mr);
1666 struct qlnxr_dev *dev = get_qlnxr_dev((ib_mr->device));
1667 int rc = 0;
1668 qlnx_host_t *ha;
1669
1670 ha = dev->ha;
1671
1672 QL_DPRINT12(ha, "enter\n");
1673
1674 if ((mr->type != QLNXR_MR_DMA) && (mr->type != QLNXR_MR_FRMR))
1675 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
1676
1677 /* it could be user registered memory. */
1678 if (mr->umem)
1679 ib_umem_release(mr->umem);
1680
1681 kfree(mr->pages);
1682
1683 kfree(mr);
1684
1685 QL_DPRINT12(ha, "exit\n");
1686 return rc;
1687 }
1688
1689 static int
1690 qlnxr_copy_cq_uresp(struct qlnxr_dev *dev,
1691 struct qlnxr_cq *cq, struct ib_udata *udata)
1692 {
1693 struct qlnxr_create_cq_uresp uresp;
1694 int rc;
1695 qlnx_host_t *ha;
1696
1697 ha = dev->ha;
1698
1699 QL_DPRINT12(ha, "enter\n");
1700
1701 memset(&uresp, 0, sizeof(uresp));
1702
1703 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
1704 uresp.icid = cq->icid;
1705
1706 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1707
1708 if (rc) {
1709 QL_DPRINT12(ha, "ib_copy_to_udata error cqid=0x%x[%d]\n",
1710 cq->icid, rc);
1711 }
1712
1713 QL_DPRINT12(ha, "exit [%d]\n", rc);
1714 return rc;
1715 }
1716
1717 static void
1718 consume_cqe(struct qlnxr_cq *cq)
1719 {
1720
1721 if (cq->latest_cqe == cq->toggle_cqe)
1722 cq->pbl_toggle ^= RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK;
1723
1724 cq->latest_cqe = ecore_chain_consume(&cq->pbl);
1725 }
1726
1727 static inline int
1728 qlnxr_align_cq_entries(int entries)
1729 {
1730 u64 size, aligned_size;
1731
1732 /* We allocate an extra entry that we don't report to the FW.
1733 * Why?
1734 * The CQE size is 32 bytes but the FW writes in chunks of 64 bytes
1735 * (for performance purposes). Allocating an extra entry and telling
1736 * the FW we have less prevents overwriting the first entry in case of
1737 * a wrap i.e. when the FW writes the last entry and the application
1738 * hasn't read the first one.
1739 */
1740 size = (entries + 1) * QLNXR_CQE_SIZE;
1741
1742 /* We align to PAGE_SIZE.
1743 * Why?
1744 * Since the CQ is going to be mapped and the mapping is anyhow in whole
1745 * kernel pages we benefit from the possibly extra CQEs.
1746 */
1747 aligned_size = ALIGN(size, PAGE_SIZE);
1748
1749 /* note: for CQs created in user space the result of this function
1750 * should match the size mapped in user space
1751 */
1752 return (aligned_size / QLNXR_CQE_SIZE);
1753 }
1754
1755 static inline int
1756 qlnxr_init_user_queue(struct ib_ucontext *ib_ctx, struct qlnxr_dev *dev,
1757 struct qlnxr_userq *q, u64 buf_addr, size_t buf_len,
1758 int access, int dmasync, int alloc_and_init)
1759 {
1760 int page_cnt;
1761 int rc;
1762 qlnx_host_t *ha;
1763
1764 ha = dev->ha;
1765
1766 QL_DPRINT12(ha, "enter\n");
1767
1768 q->buf_addr = buf_addr;
1769 q->buf_len = buf_len;
1770
1771 QL_DPRINT12(ha, "buf_addr : %llx, buf_len : %x, access : %x"
1772 " dmasync : %x\n", q->buf_addr, q->buf_len,
1773 access, dmasync);
1774
1775 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
1776
1777 if (IS_ERR(q->umem)) {
1778 QL_DPRINT11(ha, "ib_umem_get failed [%lx]\n", PTR_ERR(q->umem));
1779 return PTR_ERR(q->umem);
1780 }
1781
1782 page_cnt = ib_umem_page_count(q->umem);
1783 rc = qlnxr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt,
1784 0 /* SQ and RQ don't support dual layer pbl.
1785 * CQ may, but this is yet uncoded.
1786 */);
1787 if (rc) {
1788 QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl failed [%d]\n", rc);
1789 goto err;
1790 }
1791
1792 if (alloc_and_init) {
1793 q->pbl_tbl = qlnxr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
1794
1795 if (!q->pbl_tbl) {
1796 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n");
1797 rc = -ENOMEM;
1798 goto err;
1799 }
1800
1801 qlnxr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info);
1802 } else {
1803 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
1804
1805 if (!q->pbl_tbl) {
1806 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n");
1807 rc = -ENOMEM;
1808 goto err;
1809 }
1810 }
1811
1812 QL_DPRINT12(ha, "exit\n");
1813 return 0;
1814
1815 err:
1816 ib_umem_release(q->umem);
1817 q->umem = NULL;
1818
1819 QL_DPRINT12(ha, "exit [%d]\n", rc);
1820 return rc;
1821 }
1822
1823 #if __FreeBSD_version >= 1102000
1824
1825 struct ib_cq *
1826 qlnxr_create_cq(struct ib_device *ibdev,
1827 const struct ib_cq_init_attr *attr,
1828 struct ib_ucontext *ib_ctx,
1829 struct ib_udata *udata)
1830
1831 #else
1832
1833 #if __FreeBSD_version >= 1100000
1834
1835 struct ib_cq *
1836 qlnxr_create_cq(struct ib_device *ibdev,
1837 struct ib_cq_init_attr *attr,
1838 struct ib_ucontext *ib_ctx,
1839 struct ib_udata *udata)
1840
1841 #else
1842
1843 struct ib_cq *
1844 qlnxr_create_cq(struct ib_device *ibdev,
1845 int entries,
1846 int vector,
1847 struct ib_ucontext *ib_ctx,
1848 struct ib_udata *udata)
1849 #endif /* #if __FreeBSD_version >= 1100000 */
1850
1851 #endif /* #if __FreeBSD_version >= 1102000 */
1852 {
1853 struct qlnxr_ucontext *ctx;
1854 struct ecore_rdma_destroy_cq_out_params destroy_oparams;
1855 struct ecore_rdma_destroy_cq_in_params destroy_iparams;
1856 struct qlnxr_dev *dev;
1857 struct ecore_rdma_create_cq_in_params params;
1858 struct qlnxr_create_cq_ureq ureq;
1859
1860 #if __FreeBSD_version >= 1100000
1861 int vector = attr->comp_vector;
1862 int entries = attr->cqe;
1863 #endif
1864 struct qlnxr_cq *cq;
1865 int chain_entries, rc, page_cnt;
1866 u64 pbl_ptr;
1867 u16 icid;
1868 qlnx_host_t *ha;
1869
1870 dev = get_qlnxr_dev(ibdev);
1871 ha = dev->ha;
1872
1873 QL_DPRINT12(ha, "called from %s. entries = %d, "
1874 "vector = %d\n",
1875 (udata ? "User Lib" : "Kernel"), entries, vector);
1876
1877 memset(¶ms, 0, sizeof(struct ecore_rdma_create_cq_in_params));
1878 memset(&destroy_iparams, 0, sizeof(struct ecore_rdma_destroy_cq_in_params));
1879 memset(&destroy_oparams, 0, sizeof(struct ecore_rdma_destroy_cq_out_params));
1880
1881 if (entries > QLNXR_MAX_CQES) {
1882 QL_DPRINT11(ha,
1883 "the number of entries %d is too high. "
1884 "Must be equal or below %d.\n",
1885 entries, QLNXR_MAX_CQES);
1886 return ERR_PTR(-EINVAL);
1887 }
1888 chain_entries = qlnxr_align_cq_entries(entries);
1889 chain_entries = min_t(int, chain_entries, QLNXR_MAX_CQES);
1890
1891 cq = qlnx_zalloc((sizeof(struct qlnxr_cq)));
1892
1893 if (!cq)
1894 return ERR_PTR(-ENOMEM);
1895
1896 if (udata) {
1897 memset(&ureq, 0, sizeof(ureq));
1898
1899 if (ib_copy_from_udata(&ureq, udata,
1900 min(sizeof(ureq), udata->inlen))) {
1901 QL_DPRINT11(ha, "ib_copy_from_udata failed\n");
1902 goto err0;
1903 }
1904
1905 if (!ureq.len) {
1906 QL_DPRINT11(ha, "ureq.len == 0\n");
1907 goto err0;
1908 }
1909
1910 cq->cq_type = QLNXR_CQ_TYPE_USER;
1911
1912 qlnxr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr, ureq.len,
1913 IB_ACCESS_LOCAL_WRITE, 1, 1);
1914
1915 pbl_ptr = cq->q.pbl_tbl->pa;
1916 page_cnt = cq->q.pbl_info.num_pbes;
1917 cq->ibcq.cqe = chain_entries;
1918 } else {
1919 cq->cq_type = QLNXR_CQ_TYPE_KERNEL;
1920
1921 rc = ecore_chain_alloc(&dev->ha->cdev,
1922 ECORE_CHAIN_USE_TO_CONSUME,
1923 ECORE_CHAIN_MODE_PBL,
1924 ECORE_CHAIN_CNT_TYPE_U32,
1925 chain_entries,
1926 sizeof(union roce_cqe),
1927 &cq->pbl, NULL);
1928
1929 if (rc)
1930 goto err1;
1931
1932 page_cnt = ecore_chain_get_page_cnt(&cq->pbl);
1933 pbl_ptr = ecore_chain_get_pbl_phys(&cq->pbl);
1934 cq->ibcq.cqe = cq->pbl.capacity;
1935 }
1936
1937 params.cq_handle_hi = upper_32_bits((uintptr_t)cq);
1938 params.cq_handle_lo = lower_32_bits((uintptr_t)cq);
1939 params.cnq_id = vector;
1940 params.cq_size = chain_entries - 1;
1941 params.pbl_num_pages = page_cnt;
1942 params.pbl_ptr = pbl_ptr;
1943 params.pbl_two_level = 0;
1944
1945 if (ib_ctx != NULL) {
1946 ctx = get_qlnxr_ucontext(ib_ctx);
1947 params.dpi = ctx->dpi;
1948 } else {
1949 params.dpi = dev->dpi;
1950 }
1951
1952 rc = ecore_rdma_create_cq(dev->rdma_ctx, ¶ms, &icid);
1953 if (rc)
1954 goto err2;
1955
1956 cq->icid = icid;
1957 cq->sig = QLNXR_CQ_MAGIC_NUMBER;
1958 spin_lock_init(&cq->cq_lock);
1959
1960 if (ib_ctx) {
1961 rc = qlnxr_copy_cq_uresp(dev, cq, udata);
1962 if (rc)
1963 goto err3;
1964 } else {
1965 /* Generate doorbell address.
1966 * Configure bits 3-9 with DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT.
1967 * TODO: consider moving to device scope as it is a function of
1968 * the device.
1969 * TODO: add ifdef if plan to support 16 bit.
1970 */
1971 cq->db_addr = dev->db_addr +
1972 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
1973 cq->db.data.icid = cq->icid;
1974 cq->db.data.params = DB_AGG_CMD_SET <<
1975 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1976
1977 /* point to the very last element, passing it we will toggle */
1978 cq->toggle_cqe = ecore_chain_get_last_elem(&cq->pbl);
1979 cq->pbl_toggle = RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK;
1980
1981 /* must be different from pbl_toggle */
1982 cq->latest_cqe = NULL;
1983 consume_cqe(cq);
1984 cq->cq_cons = ecore_chain_get_cons_idx_u32(&cq->pbl);
1985 }
1986
1987 QL_DPRINT12(ha, "exit icid = 0x%0x, addr = %p,"
1988 " number of entries = 0x%x\n",
1989 cq->icid, cq, params.cq_size);
1990 QL_DPRINT12(ha,"cq_addr = %p\n", cq);
1991 return &cq->ibcq;
1992
1993 err3:
1994 destroy_iparams.icid = cq->icid;
1995 ecore_rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, &destroy_oparams);
1996 err2:
1997 if (udata)
1998 qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1999 else
2000 ecore_chain_free(&dev->ha->cdev, &cq->pbl);
2001 err1:
2002 if (udata)
2003 ib_umem_release(cq->q.umem);
2004 err0:
2005 kfree(cq);
2006
2007 QL_DPRINT12(ha, "exit error\n");
2008
2009 return ERR_PTR(-EINVAL);
2010 }
2011
2012 int qlnxr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
2013 {
2014 int status = 0;
2015 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device));
2016 qlnx_host_t *ha;
2017
2018 ha = dev->ha;
2019
2020 QL_DPRINT12(ha, "enter/exit\n");
2021
2022 return status;
2023 }
2024
2025 int
2026 qlnxr_destroy_cq(struct ib_cq *ibcq)
2027 {
2028 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device));
2029 struct ecore_rdma_destroy_cq_out_params oparams;
2030 struct ecore_rdma_destroy_cq_in_params iparams;
2031 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq);
2032 int rc = 0;
2033 qlnx_host_t *ha;
2034
2035 ha = dev->ha;
2036
2037 QL_DPRINT12(ha, "enter cq_id = %d\n", cq->icid);
2038
2039 cq->destroyed = 1;
2040
2041 /* TODO: Syncronize irq of the CNQ the CQ belongs to for validation
2042 * that all completions with notification are dealt with. The rest
2043 * of the completions are not interesting
2044 */
2045
2046 /* GSIs CQs are handled by driver, so they don't exist in the FW */
2047
2048 if (cq->cq_type != QLNXR_CQ_TYPE_GSI) {
2049 iparams.icid = cq->icid;
2050
2051 rc = ecore_rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
2052
2053 if (rc) {
2054 QL_DPRINT12(ha, "ecore_rdma_destroy_cq failed cq_id = %d\n",
2055 cq->icid);
2056 return rc;
2057 }
2058
2059 QL_DPRINT12(ha, "free cq->pbl cq_id = %d\n", cq->icid);
2060 ecore_chain_free(&dev->ha->cdev, &cq->pbl);
2061 }
2062
2063 if (ibcq->uobject && ibcq->uobject->context) {
2064 qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
2065 ib_umem_release(cq->q.umem);
2066 }
2067
2068 cq->sig = ~cq->sig;
2069
2070 kfree(cq);
2071
2072 QL_DPRINT12(ha, "exit cq_id = %d\n", cq->icid);
2073
2074 return rc;
2075 }
2076
2077 static int
2078 qlnxr_check_qp_attrs(struct ib_pd *ibpd,
2079 struct qlnxr_dev *dev,
2080 struct ib_qp_init_attr *attrs,
2081 struct ib_udata *udata)
2082 {
2083 struct ecore_rdma_device *qattr;
2084 qlnx_host_t *ha;
2085
2086 qattr = ecore_rdma_query_device(dev->rdma_ctx);
2087 ha = dev->ha;
2088
2089 QL_DPRINT12(ha, "enter\n");
2090
2091 QL_DPRINT12(ha, "attrs->sq_sig_type = %d\n", attrs->sq_sig_type);
2092 QL_DPRINT12(ha, "attrs->qp_type = %d\n", attrs->qp_type);
2093 QL_DPRINT12(ha, "attrs->create_flags = %d\n", attrs->create_flags);
2094
2095 #if __FreeBSD_version < 1102000
2096 QL_DPRINT12(ha, "attrs->qpg_type = %d\n", attrs->qpg_type);
2097 #endif
2098
2099 QL_DPRINT12(ha, "attrs->port_num = %d\n", attrs->port_num);
2100 QL_DPRINT12(ha, "attrs->cap.max_send_wr = 0x%x\n", attrs->cap.max_send_wr);
2101 QL_DPRINT12(ha, "attrs->cap.max_recv_wr = 0x%x\n", attrs->cap.max_recv_wr);
2102 QL_DPRINT12(ha, "attrs->cap.max_send_sge = 0x%x\n", attrs->cap.max_send_sge);
2103 QL_DPRINT12(ha, "attrs->cap.max_recv_sge = 0x%x\n", attrs->cap.max_recv_sge);
2104 QL_DPRINT12(ha, "attrs->cap.max_inline_data = 0x%x\n",
2105 attrs->cap.max_inline_data);
2106
2107 #if __FreeBSD_version < 1102000
2108 QL_DPRINT12(ha, "attrs->cap.qpg_tss_mask_sz = 0x%x\n",
2109 attrs->cap.qpg_tss_mask_sz);
2110 #endif
2111
2112 QL_DPRINT12(ha, "\n\nqattr->vendor_id = 0x%x\n", qattr->vendor_id);
2113 QL_DPRINT12(ha, "qattr->vendor_part_id = 0x%x\n", qattr->vendor_part_id);
2114 QL_DPRINT12(ha, "qattr->hw_ver = 0x%x\n", qattr->hw_ver);
2115 QL_DPRINT12(ha, "qattr->fw_ver = %p\n", (void *)qattr->fw_ver);
2116 QL_DPRINT12(ha, "qattr->node_guid = %p\n", (void *)qattr->node_guid);
2117 QL_DPRINT12(ha, "qattr->sys_image_guid = %p\n",
2118 (void *)qattr->sys_image_guid);
2119 QL_DPRINT12(ha, "qattr->max_cnq = 0x%x\n", qattr->max_cnq);
2120 QL_DPRINT12(ha, "qattr->max_sge = 0x%x\n", qattr->max_sge);
2121 QL_DPRINT12(ha, "qattr->max_srq_sge = 0x%x\n", qattr->max_srq_sge);
2122 QL_DPRINT12(ha, "qattr->max_inline = 0x%x\n", qattr->max_inline);
2123 QL_DPRINT12(ha, "qattr->max_wqe = 0x%x\n", qattr->max_wqe);
2124 QL_DPRINT12(ha, "qattr->max_srq_wqe = 0x%x\n", qattr->max_srq_wqe);
2125 QL_DPRINT12(ha, "qattr->max_qp_resp_rd_atomic_resc = 0x%x\n",
2126 qattr->max_qp_resp_rd_atomic_resc);
2127 QL_DPRINT12(ha, "qattr->max_qp_req_rd_atomic_resc = 0x%x\n",
2128 qattr->max_qp_req_rd_atomic_resc);
2129 QL_DPRINT12(ha, "qattr->max_dev_resp_rd_atomic_resc = 0x%x\n",
2130 qattr->max_dev_resp_rd_atomic_resc);
2131 QL_DPRINT12(ha, "qattr->max_cq = 0x%x\n", qattr->max_cq);
2132 QL_DPRINT12(ha, "qattr->max_qp = 0x%x\n", qattr->max_qp);
2133 QL_DPRINT12(ha, "qattr->max_srq = 0x%x\n", qattr->max_srq);
2134 QL_DPRINT12(ha, "qattr->max_mr = 0x%x\n", qattr->max_mr);
2135 QL_DPRINT12(ha, "qattr->max_mr_size = %p\n", (void *)qattr->max_mr_size);
2136 QL_DPRINT12(ha, "qattr->max_cqe = 0x%x\n", qattr->max_cqe);
2137 QL_DPRINT12(ha, "qattr->max_mw = 0x%x\n", qattr->max_mw);
2138 QL_DPRINT12(ha, "qattr->max_fmr = 0x%x\n", qattr->max_fmr);
2139 QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_pbl = 0x%x\n",
2140 qattr->max_mr_mw_fmr_pbl);
2141 QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_size = %p\n",
2142 (void *)qattr->max_mr_mw_fmr_size);
2143 QL_DPRINT12(ha, "qattr->max_pd = 0x%x\n", qattr->max_pd);
2144 QL_DPRINT12(ha, "qattr->max_ah = 0x%x\n", qattr->max_ah);
2145 QL_DPRINT12(ha, "qattr->max_pkey = 0x%x\n", qattr->max_pkey);
2146 QL_DPRINT12(ha, "qattr->max_srq_wr = 0x%x\n", qattr->max_srq_wr);
2147 QL_DPRINT12(ha, "qattr->max_stats_queues = 0x%x\n",
2148 qattr->max_stats_queues);
2149 //QL_DPRINT12(ha, "qattr->dev_caps = 0x%x\n", qattr->dev_caps);
2150 QL_DPRINT12(ha, "qattr->page_size_caps = %p\n",
2151 (void *)qattr->page_size_caps);
2152 QL_DPRINT12(ha, "qattr->dev_ack_delay = 0x%x\n", qattr->dev_ack_delay);
2153 QL_DPRINT12(ha, "qattr->reserved_lkey = 0x%x\n", qattr->reserved_lkey);
2154 QL_DPRINT12(ha, "qattr->bad_pkey_counter = 0x%x\n",
2155 qattr->bad_pkey_counter);
2156
2157 if ((attrs->qp_type == IB_QPT_GSI) && udata) {
2158 QL_DPRINT12(ha, "unexpected udata when creating GSI QP\n");
2159 return -EINVAL;
2160 }
2161
2162 if (udata && !(ibpd->uobject && ibpd->uobject->context)) {
2163 QL_DPRINT12(ha, "called from user without context\n");
2164 return -EINVAL;
2165 }
2166
2167 /* QP0... attrs->qp_type == IB_QPT_GSI */
2168 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
2169 QL_DPRINT12(ha, "unsupported qp type=0x%x requested\n",
2170 attrs->qp_type);
2171 return -EINVAL;
2172 }
2173 if (attrs->qp_type == IB_QPT_GSI && attrs->srq) {
2174 QL_DPRINT12(ha, "cannot create GSI qp with SRQ\n");
2175 return -EINVAL;
2176 }
2177 /* Skip the check for QP1 to support CM size of 128 */
2178 if (attrs->cap.max_send_wr > qattr->max_wqe) {
2179 QL_DPRINT12(ha, "cannot create a SQ with %d elements "
2180 " (max_send_wr=0x%x)\n",
2181 attrs->cap.max_send_wr, qattr->max_wqe);
2182 return -EINVAL;
2183 }
2184 if (!attrs->srq && (attrs->cap.max_recv_wr > qattr->max_wqe)) {
2185 QL_DPRINT12(ha, "cannot create a RQ with %d elements"
2186 " (max_recv_wr=0x%x)\n",
2187 attrs->cap.max_recv_wr, qattr->max_wqe);
2188 return -EINVAL;
2189 }
2190 if (attrs->cap.max_inline_data > qattr->max_inline) {
2191 QL_DPRINT12(ha,
2192 "unsupported inline data size=0x%x "
2193 "requested (max_inline=0x%x)\n",
2194 attrs->cap.max_inline_data, qattr->max_inline);
2195 return -EINVAL;
2196 }
2197 if (attrs->cap.max_send_sge > qattr->max_sge) {
2198 QL_DPRINT12(ha,
2199 "unsupported send_sge=0x%x "
2200 "requested (max_send_sge=0x%x)\n",
2201 attrs->cap.max_send_sge, qattr->max_sge);
2202 return -EINVAL;
2203 }
2204 if (attrs->cap.max_recv_sge > qattr->max_sge) {
2205 QL_DPRINT12(ha,
2206 "unsupported recv_sge=0x%x requested "
2207 " (max_recv_sge=0x%x)\n",
2208 attrs->cap.max_recv_sge, qattr->max_sge);
2209 return -EINVAL;
2210 }
2211 /* unprivileged user space cannot create special QP */
2212 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
2213 QL_DPRINT12(ha,
2214 "userspace can't create special QPs of type=0x%x\n",
2215 attrs->qp_type);
2216 return -EINVAL;
2217 }
2218 /* allow creating only one GSI type of QP */
2219 if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) {
2220 QL_DPRINT12(ha,
2221 "create qp: GSI special QPs already created.\n");
2222 return -EINVAL;
2223 }
2224
2225 /* verify consumer QPs are not trying to use GSI QP's CQ */
2226 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) {
2227 struct qlnxr_cq *send_cq = get_qlnxr_cq(attrs->send_cq);
2228 struct qlnxr_cq *recv_cq = get_qlnxr_cq(attrs->recv_cq);
2229
2230 if ((send_cq->cq_type == QLNXR_CQ_TYPE_GSI) ||
2231 (recv_cq->cq_type == QLNXR_CQ_TYPE_GSI)) {
2232 QL_DPRINT11(ha, "consumer QP cannot use GSI CQs.\n");
2233 return -EINVAL;
2234 }
2235 }
2236 QL_DPRINT12(ha, "exit\n");
2237 return 0;
2238 }
2239
2240 static int
2241 qlnxr_copy_srq_uresp(struct qlnxr_dev *dev,
2242 struct qlnxr_srq *srq,
2243 struct ib_udata *udata)
2244 {
2245 struct qlnxr_create_srq_uresp uresp;
2246 qlnx_host_t *ha;
2247 int rc;
2248
2249 ha = dev->ha;
2250
2251 QL_DPRINT12(ha, "enter\n");
2252
2253 memset(&uresp, 0, sizeof(uresp));
2254
2255 uresp.srq_id = srq->srq_id;
2256
2257 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
2258
2259 QL_DPRINT12(ha, "exit [%d]\n", rc);
2260 return rc;
2261 }
2262
2263 static void
2264 qlnxr_copy_rq_uresp(struct qlnxr_dev *dev,
2265 struct qlnxr_create_qp_uresp *uresp,
2266 struct qlnxr_qp *qp)
2267 {
2268 qlnx_host_t *ha;
2269
2270 ha = dev->ha;
2271
2272 /* Return if QP is associated with SRQ instead of RQ */
2273 QL_DPRINT12(ha, "enter qp->srq = %p\n", qp->srq);
2274
2275 if (qp->srq)
2276 return;
2277
2278 /* iWARP requires two doorbells per RQ. */
2279 if (QLNX_IS_IWARP(dev)) {
2280 uresp->rq_db_offset =
2281 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
2282 uresp->rq_db2_offset =
2283 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
2284
2285 QL_DPRINT12(ha, "uresp->rq_db_offset = 0x%x "
2286 "uresp->rq_db2_offset = 0x%x\n",
2287 uresp->rq_db_offset, uresp->rq_db2_offset);
2288 } else {
2289 uresp->rq_db_offset =
2290 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
2291 }
2292 uresp->rq_icid = qp->icid;
2293
2294 QL_DPRINT12(ha, "exit\n");
2295 return;
2296 }
2297
2298 static void
2299 qlnxr_copy_sq_uresp(struct qlnxr_dev *dev,
2300 struct qlnxr_create_qp_uresp *uresp,
2301 struct qlnxr_qp *qp)
2302 {
2303 qlnx_host_t *ha;
2304
2305 ha = dev->ha;
2306
2307 QL_DPRINT12(ha, "enter\n");
2308
2309 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
2310
2311 /* iWARP uses the same cid for rq and sq*/
2312 if (QLNX_IS_IWARP(dev)) {
2313 uresp->sq_icid = qp->icid;
2314 QL_DPRINT12(ha, "uresp->sq_icid = 0x%x\n", uresp->sq_icid);
2315 } else
2316 uresp->sq_icid = qp->icid + 1;
2317
2318 QL_DPRINT12(ha, "exit\n");
2319 return;
2320 }
2321
2322 static int
2323 qlnxr_copy_qp_uresp(struct qlnxr_dev *dev,
2324 struct qlnxr_qp *qp,
2325 struct ib_udata *udata)
2326 {
2327 int rc;
2328 struct qlnxr_create_qp_uresp uresp;
2329 qlnx_host_t *ha;
2330
2331 ha = dev->ha;
2332
2333 QL_DPRINT12(ha, "enter qp->icid =0x%x\n", qp->icid);
2334
2335 memset(&uresp, 0, sizeof(uresp));
2336 qlnxr_copy_sq_uresp(dev, &uresp, qp);
2337 qlnxr_copy_rq_uresp(dev, &uresp, qp);
2338
2339 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
2340 uresp.qp_id = qp->qp_id;
2341
2342 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
2343
2344 QL_DPRINT12(ha, "exit [%d]\n", rc);
2345 return rc;
2346 }
2347
2348 static void
2349 qlnxr_set_common_qp_params(struct qlnxr_dev *dev,
2350 struct qlnxr_qp *qp,
2351 struct qlnxr_pd *pd,
2352 struct ib_qp_init_attr *attrs)
2353 {
2354 qlnx_host_t *ha;
2355
2356 ha = dev->ha;
2357
2358 QL_DPRINT12(ha, "enter\n");
2359
2360 spin_lock_init(&qp->q_lock);
2361
2362 atomic_set(&qp->refcnt, 1);
2363 qp->pd = pd;
2364 qp->sig = QLNXR_QP_MAGIC_NUMBER;
2365 qp->qp_type = attrs->qp_type;
2366 qp->max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
2367 qp->sq.max_sges = attrs->cap.max_send_sge;
2368 qp->state = ECORE_ROCE_QP_STATE_RESET;
2369 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
2370 qp->sq_cq = get_qlnxr_cq(attrs->send_cq);
2371 qp->rq_cq = get_qlnxr_cq(attrs->recv_cq);
2372 qp->dev = dev;
2373
2374 if (!attrs->srq) {
2375 /* QP is associated with RQ instead of SRQ */
2376 qp->rq.max_sges = attrs->cap.max_recv_sge;
2377 QL_DPRINT12(ha, "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
2378 qp->rq.max_sges, qp->rq_cq->icid);
2379 } else {
2380 qp->srq = get_qlnxr_srq(attrs->srq);
2381 }
2382
2383 QL_DPRINT12(ha,
2384 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d,"
2385 " state = %d, signaled = %d, use_srq=%d\n",
2386 pd->pd_id, qp->qp_type, qp->max_inline_data,
2387 qp->state, qp->signaled, ((attrs->srq) ? 1 : 0));
2388 QL_DPRINT12(ha, "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
2389 qp->sq.max_sges, qp->sq_cq->icid);
2390 return;
2391 }
2392
2393 static int
2394 qlnxr_check_srq_params(struct ib_pd *ibpd,
2395 struct qlnxr_dev *dev,
2396 struct ib_srq_init_attr *attrs)
2397 {
2398 struct ecore_rdma_device *qattr;
2399 qlnx_host_t *ha;
2400
2401 ha = dev->ha;
2402 qattr = ecore_rdma_query_device(dev->rdma_ctx);
2403
2404 QL_DPRINT12(ha, "enter\n");
2405
2406 if (attrs->attr.max_wr > qattr->max_srq_wqe) {
2407 QL_DPRINT12(ha, "unsupported srq_wr=0x%x"
2408 " requested (max_srq_wr=0x%x)\n",
2409 attrs->attr.max_wr, qattr->max_srq_wr);
2410 return -EINVAL;
2411 }
2412
2413 if (attrs->attr.max_sge > qattr->max_sge) {
2414 QL_DPRINT12(ha,
2415 "unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
2416 attrs->attr.max_sge, qattr->max_sge);
2417 return -EINVAL;
2418 }
2419
2420 if (attrs->attr.srq_limit > attrs->attr.max_wr) {
2421 QL_DPRINT12(ha,
2422 "unsupported srq_limit=0x%x requested"
2423 " (max_srq_limit=0x%x)\n",
2424 attrs->attr.srq_limit, attrs->attr.srq_limit);
2425 return -EINVAL;
2426 }
2427
2428 QL_DPRINT12(ha, "exit\n");
2429 return 0;
2430 }
2431
2432 static void
2433 qlnxr_free_srq_user_params(struct qlnxr_srq *srq)
2434 {
2435 struct qlnxr_dev *dev = srq->dev;
2436 qlnx_host_t *ha;
2437
2438 ha = dev->ha;
2439
2440 QL_DPRINT12(ha, "enter\n");
2441
2442 qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
2443 ib_umem_release(srq->usrq.umem);
2444 ib_umem_release(srq->prod_umem);
2445
2446 QL_DPRINT12(ha, "exit\n");
2447 return;
2448 }
2449
2450 static void
2451 qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq)
2452 {
2453 struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq;
2454 struct qlnxr_dev *dev = srq->dev;
2455 qlnx_host_t *ha;
2456
2457 ha = dev->ha;
2458
2459 QL_DPRINT12(ha, "enter\n");
2460
2461 ecore_chain_free(dev->cdev, &hw_srq->pbl);
2462
2463 qlnx_dma_free_coherent(&dev->cdev,
2464 hw_srq->virt_prod_pair_addr,
2465 hw_srq->phy_prod_pair_addr,
2466 sizeof(struct rdma_srq_producers));
2467
2468 QL_DPRINT12(ha, "exit\n");
2469
2470 return;
2471 }
2472
2473 static int
2474 qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx,
2475 struct qlnxr_srq *srq,
2476 struct qlnxr_create_srq_ureq *ureq,
2477 int access, int dmasync)
2478 {
2479 #ifdef DEFINE_IB_UMEM_WITH_CHUNK
2480 struct ib_umem_chunk *chunk;
2481 #endif
2482 struct scatterlist *sg;
2483 int rc;
2484 struct qlnxr_dev *dev = srq->dev;
2485 qlnx_host_t *ha;
2486
2487 ha = dev->ha;
2488
2489 QL_DPRINT12(ha, "enter\n");
2490
2491 rc = qlnxr_init_user_queue(ib_ctx, srq->dev, &srq->usrq, ureq->srq_addr,
2492 ureq->srq_len, access, dmasync, 1);
2493 if (rc)
2494 return rc;
2495
2496 srq->prod_umem = ib_umem_get(ib_ctx, ureq->prod_pair_addr,
2497 sizeof(struct rdma_srq_producers),
2498 access, dmasync);
2499 if (IS_ERR(srq->prod_umem)) {
2500 qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
2501 ib_umem_release(srq->usrq.umem);
2502
2503 QL_DPRINT12(ha, "ib_umem_get failed for producer [%p]\n",
2504 PTR_ERR(srq->prod_umem));
2505
2506 return PTR_ERR(srq->prod_umem);
2507 }
2508
2509 #ifdef DEFINE_IB_UMEM_WITH_CHUNK
2510 chunk = container_of((&srq->prod_umem->chunk_list)->next,
2511 typeof(*chunk), list);
2512 sg = &chunk->page_list[0];
2513 #else
2514 sg = srq->prod_umem->sg_head.sgl;
2515 #endif
2516 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
2517
2518 QL_DPRINT12(ha, "exit\n");
2519 return 0;
2520 }
2521
2522 static int
2523 qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq,
2524 struct qlnxr_dev *dev,
2525 struct ib_srq_init_attr *init_attr)
2526 {
2527 struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq;
2528 dma_addr_t phy_prod_pair_addr;
2529 u32 num_elems, max_wr;
2530 void *va;
2531 int rc;
2532 qlnx_host_t *ha;
2533
2534 ha = dev->ha;
2535
2536 QL_DPRINT12(ha, "enter\n");
2537
2538 va = qlnx_dma_alloc_coherent(&dev->cdev,
2539 &phy_prod_pair_addr,
2540 sizeof(struct rdma_srq_producers));
2541 if (!va) {
2542 QL_DPRINT11(ha, "qlnx_dma_alloc_coherent failed for produceer\n");
2543 return -ENOMEM;
2544 }
2545
2546 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
2547 hw_srq->virt_prod_pair_addr = va;
2548
2549 max_wr = init_attr->attr.max_wr;
2550
2551 num_elems = max_wr * RDMA_MAX_SRQ_WQE_SIZE;
2552
2553 rc = ecore_chain_alloc(dev->cdev,
2554 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
2555 ECORE_CHAIN_MODE_PBL,
2556 ECORE_CHAIN_CNT_TYPE_U32,
2557 num_elems,
2558 ECORE_RDMA_SRQ_WQE_ELEM_SIZE,
2559 &hw_srq->pbl, NULL);
2560
2561 if (rc) {
2562 QL_DPRINT11(ha, "ecore_chain_alloc failed [%d]\n", rc);
2563 goto err0;
2564 }
2565
2566 hw_srq->max_wr = max_wr;
2567 hw_srq->num_elems = num_elems;
2568 hw_srq->max_sges = RDMA_MAX_SGE_PER_SRQ;
2569
2570 QL_DPRINT12(ha, "exit\n");
2571 return 0;
2572
2573 err0:
2574 qlnx_dma_free_coherent(&dev->cdev, va, phy_prod_pair_addr,
2575 sizeof(struct rdma_srq_producers));
2576
2577 QL_DPRINT12(ha, "exit [%d]\n", rc);
2578 return rc;
2579 }
2580
2581 static inline void
2582 qlnxr_init_common_qp_in_params(struct qlnxr_dev *dev,
2583 struct qlnxr_pd *pd,
2584 struct qlnxr_qp *qp,
2585 struct ib_qp_init_attr *attrs,
2586 bool fmr_and_reserved_lkey,
2587 struct ecore_rdma_create_qp_in_params *params)
2588 {
2589 qlnx_host_t *ha;
2590
2591 ha = dev->ha;
2592
2593 QL_DPRINT12(ha, "enter\n");
2594
2595 /* QP handle to be written in an async event */
2596 params->qp_handle_async_lo = lower_32_bits((uintptr_t)qp);
2597 params->qp_handle_async_hi = upper_32_bits((uintptr_t)qp);
2598
2599 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
2600 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
2601 params->pd = pd->pd_id;
2602 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
2603 params->sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid;
2604 params->stats_queue = 0;
2605
2606 params->rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid;
2607
2608 if (qp->srq) {
2609 /* QP is associated with SRQ instead of RQ */
2610 params->srq_id = qp->srq->srq_id;
2611 params->use_srq = true;
2612 QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n",
2613 params->srq_id, params->use_srq);
2614 return;
2615 }
2616
2617 params->srq_id = 0;
2618 params->use_srq = false;
2619
2620 QL_DPRINT12(ha, "exit\n");
2621 return;
2622 }
2623
2624 static inline void
2625 qlnxr_qp_user_print( struct qlnxr_dev *dev,
2626 struct qlnxr_qp *qp)
2627 {
2628 QL_DPRINT12((dev->ha), "qp=%p. sq_addr=0x%llx, sq_len=%zd, "
2629 "rq_addr=0x%llx, rq_len=%zd\n",
2630 qp, qp->usq.buf_addr, qp->usq.buf_len, qp->urq.buf_addr,
2631 qp->urq.buf_len);
2632 return;
2633 }
2634
2635 static int
2636 qlnxr_idr_add(struct qlnxr_dev *dev, void *ptr, u32 id)
2637 {
2638 u32 newid;
2639 int rc;
2640 qlnx_host_t *ha;
2641
2642 ha = dev->ha;
2643
2644 QL_DPRINT12(ha, "enter\n");
2645
2646 if (!QLNX_IS_IWARP(dev))
2647 return 0;
2648
2649 do {
2650 if (!idr_pre_get(&dev->qpidr, GFP_KERNEL)) {
2651 QL_DPRINT11(ha, "idr_pre_get failed\n");
2652 return -ENOMEM;
2653 }
2654
2655 mtx_lock(&dev->idr_lock);
2656
2657 rc = idr_get_new_above(&dev->qpidr, ptr, id, &newid);
2658
2659 mtx_unlock(&dev->idr_lock);
2660
2661 } while (rc == -EAGAIN);
2662
2663 QL_DPRINT12(ha, "exit [%d]\n", rc);
2664
2665 return rc;
2666 }
2667
2668 static void
2669 qlnxr_idr_remove(struct qlnxr_dev *dev, u32 id)
2670 {
2671 qlnx_host_t *ha;
2672
2673 ha = dev->ha;
2674
2675 QL_DPRINT12(ha, "enter\n");
2676
2677 if (!QLNX_IS_IWARP(dev))
2678 return;
2679
2680 mtx_lock(&dev->idr_lock);
2681 idr_remove(&dev->qpidr, id);
2682 mtx_unlock(&dev->idr_lock);
2683
2684 QL_DPRINT12(ha, "exit \n");
2685
2686 return;
2687 }
2688
2689 static inline void
2690 qlnxr_iwarp_populate_user_qp(struct qlnxr_dev *dev,
2691 struct qlnxr_qp *qp,
2692 struct ecore_rdma_create_qp_out_params *out_params)
2693 {
2694 qlnx_host_t *ha;
2695
2696 ha = dev->ha;
2697
2698 QL_DPRINT12(ha, "enter\n");
2699
2700 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
2701 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
2702
2703 qlnxr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
2704 &qp->usq.pbl_info);
2705
2706 if (qp->srq) {
2707 QL_DPRINT11(ha, "qp->srq = %p\n", qp->srq);
2708 return;
2709 }
2710
2711 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
2712 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
2713
2714 qlnxr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
2715 &qp->urq.pbl_info);
2716
2717 QL_DPRINT12(ha, "exit\n");
2718 return;
2719 }
2720
2721 static int
2722 qlnxr_create_user_qp(struct qlnxr_dev *dev,
2723 struct qlnxr_qp *qp,
2724 struct ib_pd *ibpd,
2725 struct ib_udata *udata,
2726 struct ib_qp_init_attr *attrs)
2727 {
2728 struct ecore_rdma_destroy_qp_out_params d_out_params;
2729 struct ecore_rdma_create_qp_in_params in_params;
2730 struct ecore_rdma_create_qp_out_params out_params;
2731 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
2732 struct ib_ucontext *ib_ctx = NULL;
2733 struct qlnxr_ucontext *ctx = NULL;
2734 struct qlnxr_create_qp_ureq ureq;
2735 int alloc_and_init = QLNX_IS_ROCE(dev);
2736 int rc = -EINVAL;
2737 qlnx_host_t *ha;
2738
2739 ha = dev->ha;
2740
2741 QL_DPRINT12(ha, "enter\n");
2742
2743 ib_ctx = ibpd->uobject->context;
2744 ctx = get_qlnxr_ucontext(ib_ctx);
2745
2746 memset(&ureq, 0, sizeof(ureq));
2747 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
2748
2749 if (rc) {
2750 QL_DPRINT11(ha, "ib_copy_from_udata failed [%d]\n", rc);
2751 return rc;
2752 }
2753
2754 /* SQ - read access only (0), dma sync not required (0) */
2755 rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr,
2756 ureq.sq_len, 0, 0,
2757 alloc_and_init);
2758 if (rc) {
2759 QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc);
2760 return rc;
2761 }
2762
2763 if (!qp->srq) {
2764 /* RQ - read access only (0), dma sync not required (0) */
2765 rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr,
2766 ureq.rq_len, 0, 0,
2767 alloc_and_init);
2768
2769 if (rc) {
2770 QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc);
2771 return rc;
2772 }
2773 }
2774
2775 memset(&in_params, 0, sizeof(in_params));
2776 qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
2777 in_params.qp_handle_lo = ureq.qp_handle_lo;
2778 in_params.qp_handle_hi = ureq.qp_handle_hi;
2779 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
2780 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
2781
2782 if (!qp->srq) {
2783 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
2784 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
2785 }
2786
2787 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, &in_params, &out_params);
2788
2789 if (!qp->ecore_qp) {
2790 rc = -ENOMEM;
2791 QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n");
2792 goto err1;
2793 }
2794
2795 if (QLNX_IS_IWARP(dev))
2796 qlnxr_iwarp_populate_user_qp(dev, qp, &out_params);
2797
2798 qp->qp_id = out_params.qp_id;
2799 qp->icid = out_params.icid;
2800
2801 rc = qlnxr_copy_qp_uresp(dev, qp, udata);
2802
2803 if (rc) {
2804 QL_DPRINT11(ha, "qlnxr_copy_qp_uresp failed\n");
2805 goto err;
2806 }
2807
2808 qlnxr_qp_user_print(dev, qp);
2809
2810 QL_DPRINT12(ha, "exit\n");
2811 return 0;
2812 err:
2813 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params);
2814
2815 if (rc)
2816 QL_DPRINT12(ha, "fatal fault\n");
2817
2818 err1:
2819 qlnxr_cleanup_user(dev, qp);
2820
2821 QL_DPRINT12(ha, "exit[%d]\n", rc);
2822 return rc;
2823 }
2824
2825 static void
2826 qlnxr_set_roce_db_info(struct qlnxr_dev *dev,
2827 struct qlnxr_qp *qp)
2828 {
2829 qlnx_host_t *ha;
2830
2831 ha = dev->ha;
2832
2833 QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq);
2834
2835 qp->sq.db = dev->db_addr +
2836 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
2837 qp->sq.db_data.data.icid = qp->icid + 1;
2838
2839 if (!qp->srq) {
2840 qp->rq.db = dev->db_addr +
2841 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
2842 qp->rq.db_data.data.icid = qp->icid;
2843 }
2844
2845 QL_DPRINT12(ha, "exit\n");
2846 return;
2847 }
2848
2849 static void
2850 qlnxr_set_iwarp_db_info(struct qlnxr_dev *dev,
2851 struct qlnxr_qp *qp)
2852
2853 {
2854 qlnx_host_t *ha;
2855
2856 ha = dev->ha;
2857
2858 QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq);
2859
2860 qp->sq.db = dev->db_addr +
2861 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
2862 qp->sq.db_data.data.icid = qp->icid;
2863
2864 if (!qp->srq) {
2865 qp->rq.db = dev->db_addr +
2866 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
2867 qp->rq.db_data.data.icid = qp->icid;
2868
2869 qp->rq.iwarp_db2 = dev->db_addr +
2870 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
2871 qp->rq.iwarp_db2_data.data.icid = qp->icid;
2872 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
2873 }
2874
2875 QL_DPRINT12(ha,
2876 "qp->sq.db = %p qp->sq.db_data.data.icid =0x%x\n"
2877 "\t\t\tqp->rq.db = %p qp->rq.db_data.data.icid =0x%x\n"
2878 "\t\t\tqp->rq.iwarp_db2 = %p qp->rq.iwarp_db2.data.icid =0x%x"
2879 " qp->rq.iwarp_db2.data.prod_val =0x%x\n",
2880 qp->sq.db, qp->sq.db_data.data.icid,
2881 qp->rq.db, qp->rq.db_data.data.icid,
2882 qp->rq.iwarp_db2, qp->rq.iwarp_db2_data.data.icid,
2883 qp->rq.iwarp_db2_data.data.value);
2884
2885 QL_DPRINT12(ha, "exit\n");
2886 return;
2887 }
2888
2889 static int
2890 qlnxr_roce_create_kernel_qp(struct qlnxr_dev *dev,
2891 struct qlnxr_qp *qp,
2892 struct ecore_rdma_create_qp_in_params *in_params,
2893 u32 n_sq_elems,
2894 u32 n_rq_elems)
2895 {
2896 struct ecore_rdma_create_qp_out_params out_params;
2897 int rc;
2898 qlnx_host_t *ha;
2899
2900 ha = dev->ha;
2901
2902 QL_DPRINT12(ha, "enter\n");
2903
2904 rc = ecore_chain_alloc(
2905 dev->cdev,
2906 ECORE_CHAIN_USE_TO_PRODUCE,
2907 ECORE_CHAIN_MODE_PBL,
2908 ECORE_CHAIN_CNT_TYPE_U32,
2909 n_sq_elems,
2910 QLNXR_SQE_ELEMENT_SIZE,
2911 &qp->sq.pbl,
2912 NULL);
2913
2914 if (rc) {
2915 QL_DPRINT11(ha, "ecore_chain_alloc qp->sq.pbl failed[%d]\n", rc);
2916 return rc;
2917 }
2918
2919 in_params->sq_num_pages = ecore_chain_get_page_cnt(&qp->sq.pbl);
2920 in_params->sq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->sq.pbl);
2921
2922 if (!qp->srq) {
2923 rc = ecore_chain_alloc(
2924 dev->cdev,
2925 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
2926 ECORE_CHAIN_MODE_PBL,
2927 ECORE_CHAIN_CNT_TYPE_U32,
2928 n_rq_elems,
2929 QLNXR_RQE_ELEMENT_SIZE,
2930 &qp->rq.pbl,
2931 NULL);
2932
2933 if (rc) {
2934 QL_DPRINT11(ha,
2935 "ecore_chain_alloc qp->rq.pbl failed[%d]\n", rc);
2936 return rc;
2937 }
2938
2939 in_params->rq_num_pages = ecore_chain_get_page_cnt(&qp->rq.pbl);
2940 in_params->rq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->rq.pbl);
2941 }
2942
2943 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params);
2944
2945 if (!qp->ecore_qp) {
2946 QL_DPRINT11(ha, "qp->ecore_qp == NULL\n");
2947 return -EINVAL;
2948 }
2949
2950 qp->qp_id = out_params.qp_id;
2951 qp->icid = out_params.icid;
2952
2953 qlnxr_set_roce_db_info(dev, qp);
2954
2955 QL_DPRINT12(ha, "exit\n");
2956 return 0;
2957 }
2958
2959 static int
2960 qlnxr_iwarp_create_kernel_qp(struct qlnxr_dev *dev,
2961 struct qlnxr_qp *qp,
2962 struct ecore_rdma_create_qp_in_params *in_params,
2963 u32 n_sq_elems,
2964 u32 n_rq_elems)
2965 {
2966 struct ecore_rdma_destroy_qp_out_params d_out_params;
2967 struct ecore_rdma_create_qp_out_params out_params;
2968 struct ecore_chain_ext_pbl ext_pbl;
2969 int rc;
2970 qlnx_host_t *ha;
2971
2972 ha = dev->ha;
2973
2974 QL_DPRINT12(ha, "enter\n");
2975
2976 in_params->sq_num_pages = ECORE_CHAIN_PAGE_CNT(n_sq_elems,
2977 QLNXR_SQE_ELEMENT_SIZE,
2978 ECORE_CHAIN_MODE_PBL);
2979 in_params->rq_num_pages = ECORE_CHAIN_PAGE_CNT(n_rq_elems,
2980 QLNXR_RQE_ELEMENT_SIZE,
2981 ECORE_CHAIN_MODE_PBL);
2982
2983 QL_DPRINT12(ha, "n_sq_elems = 0x%x"
2984 " n_rq_elems = 0x%x in_params\n"
2985 "\t\t\tqp_handle_lo\t\t= 0x%08x\n"
2986 "\t\t\tqp_handle_hi\t\t= 0x%08x\n"
2987 "\t\t\tqp_handle_async_lo\t\t= 0x%08x\n"
2988 "\t\t\tqp_handle_async_hi\t\t= 0x%08x\n"
2989 "\t\t\tuse_srq\t\t\t= 0x%x\n"
2990 "\t\t\tsignal_all\t\t= 0x%x\n"
2991 "\t\t\tfmr_and_reserved_lkey\t= 0x%x\n"
2992 "\t\t\tpd\t\t\t= 0x%x\n"
2993 "\t\t\tdpi\t\t\t= 0x%x\n"
2994 "\t\t\tsq_cq_id\t\t\t= 0x%x\n"
2995 "\t\t\tsq_num_pages\t\t= 0x%x\n"
2996 "\t\t\tsq_pbl_ptr\t\t= %p\n"
2997 "\t\t\tmax_sq_sges\t\t= 0x%x\n"
2998 "\t\t\trq_cq_id\t\t\t= 0x%x\n"
2999 "\t\t\trq_num_pages\t\t= 0x%x\n"
3000 "\t\t\trq_pbl_ptr\t\t= %p\n"
3001 "\t\t\tsrq_id\t\t\t= 0x%x\n"
3002 "\t\t\tstats_queue\t\t= 0x%x\n",
3003 n_sq_elems, n_rq_elems,
3004 in_params->qp_handle_lo,
3005 in_params->qp_handle_hi,
3006 in_params->qp_handle_async_lo,
3007 in_params->qp_handle_async_hi,
3008 in_params->use_srq,
3009 in_params->signal_all,
3010 in_params->fmr_and_reserved_lkey,
3011 in_params->pd,
3012 in_params->dpi,
3013 in_params->sq_cq_id,
3014 in_params->sq_num_pages,
3015 (void *)in_params->sq_pbl_ptr,
3016 in_params->max_sq_sges,
3017 in_params->rq_cq_id,
3018 in_params->rq_num_pages,
3019 (void *)in_params->rq_pbl_ptr,
3020 in_params->srq_id,
3021 in_params->stats_queue );
3022
3023 memset(&out_params, 0, sizeof (struct ecore_rdma_create_qp_out_params));
3024 memset(&ext_pbl, 0, sizeof (struct ecore_chain_ext_pbl));
3025
3026 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params);
3027
3028 if (!qp->ecore_qp) {
3029 QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n");
3030 return -EINVAL;
3031 }
3032
3033 /* Now we allocate the chain */
3034 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
3035 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
3036
3037 QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p "
3038 "ext_pbl.p_pbl_phys = %p\n",
3039 ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys);
3040
3041 rc = ecore_chain_alloc(
3042 dev->cdev,
3043 ECORE_CHAIN_USE_TO_PRODUCE,
3044 ECORE_CHAIN_MODE_PBL,
3045 ECORE_CHAIN_CNT_TYPE_U32,
3046 n_sq_elems,
3047 QLNXR_SQE_ELEMENT_SIZE,
3048 &qp->sq.pbl,
3049 &ext_pbl);
3050
3051 if (rc) {
3052 QL_DPRINT11(ha,
3053 "ecore_chain_alloc qp->sq.pbl failed rc = %d\n", rc);
3054 goto err;
3055 }
3056
3057 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
3058 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
3059
3060 QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p "
3061 "ext_pbl.p_pbl_phys = %p\n",
3062 ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys);
3063
3064 if (!qp->srq) {
3065 rc = ecore_chain_alloc(
3066 dev->cdev,
3067 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
3068 ECORE_CHAIN_MODE_PBL,
3069 ECORE_CHAIN_CNT_TYPE_U32,
3070 n_rq_elems,
3071 QLNXR_RQE_ELEMENT_SIZE,
3072 &qp->rq.pbl,
3073 &ext_pbl);
3074
3075 if (rc) {
3076 QL_DPRINT11(ha,, "ecore_chain_alloc qp->rq.pbl"
3077 " failed rc = %d\n", rc);
3078 goto err;
3079 }
3080 }
3081
3082 QL_DPRINT12(ha, "qp_id = 0x%x icid =0x%x\n",
3083 out_params.qp_id, out_params.icid);
3084
3085 qp->qp_id = out_params.qp_id;
3086 qp->icid = out_params.icid;
3087
3088 qlnxr_set_iwarp_db_info(dev, qp);
3089
3090 QL_DPRINT12(ha, "exit\n");
3091 return 0;
3092
3093 err:
3094 ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params);
3095
3096 QL_DPRINT12(ha, "exit rc = %d\n", rc);
3097 return rc;
3098 }
3099
3100 static int
3101 qlnxr_create_kernel_qp(struct qlnxr_dev *dev,
3102 struct qlnxr_qp *qp,
3103 struct ib_pd *ibpd,
3104 struct ib_qp_init_attr *attrs)
3105 {
3106 struct ecore_rdma_create_qp_in_params in_params;
3107 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
3108 int rc = -EINVAL;
3109 u32 n_rq_elems;
3110 u32 n_sq_elems;
3111 u32 n_sq_entries;
3112 struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx);
3113 qlnx_host_t *ha;
3114
3115 ha = dev->ha;
3116
3117 QL_DPRINT12(ha, "enter\n");
3118
3119 memset(&in_params, 0, sizeof(in_params));
3120
3121 /* A single work request may take up to MAX_SQ_WQE_SIZE elements in
3122 * the ring. The ring should allow at least a single WR, even if the
3123 * user requested none, due to allocation issues.
3124 * We should add an extra WR since the prod and cons indices of
3125 * wqe_wr_id are managed in such a way that the WQ is considered full
3126 * when (prod+1)%max_wr==cons. We currently don't do that because we
3127 * double the number of entries due an iSER issue that pushes far more
3128 * WRs than indicated. If we decline its ib_post_send() then we get
3129 * error prints in the dmesg we'd like to avoid.
3130 */
3131 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
3132 qattr->max_wqe);
3133
3134 qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id),
3135 GFP_KERNEL);
3136 if (!qp->wqe_wr_id) {
3137 QL_DPRINT11(ha, "failed SQ shadow memory allocation\n");
3138 return -ENOMEM;
3139 }
3140
3141 /* QP handle to be written in CQE */
3142 in_params.qp_handle_lo = lower_32_bits((uintptr_t)qp);
3143 in_params.qp_handle_hi = upper_32_bits((uintptr_t)qp);
3144
3145 /* A single work request may take up to MAX_RQ_WQE_SIZE elements in
3146 * the ring. There ring should allow at least a single WR, even if the
3147 * user requested none, due to allocation issues.
3148 */
3149 qp->rq.max_wr = (u16)max_t(u32, attrs->cap.max_recv_wr, 1);
3150
3151 /* Allocate driver internal RQ array */
3152 if (!qp->srq) {
3153 qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id),
3154 GFP_KERNEL);
3155 if (!qp->rqe_wr_id) {
3156 QL_DPRINT11(ha, "failed RQ shadow memory allocation\n");
3157 kfree(qp->wqe_wr_id);
3158 return -ENOMEM;
3159 }
3160 }
3161
3162 //qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
3163
3164 in_params.qp_handle_async_lo = lower_32_bits((uintptr_t)qp);
3165 in_params.qp_handle_async_hi = upper_32_bits((uintptr_t)qp);
3166
3167 in_params.signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
3168 in_params.fmr_and_reserved_lkey = true;
3169 in_params.pd = pd->pd_id;
3170 in_params.dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
3171 in_params.sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid;
3172 in_params.stats_queue = 0;
3173
3174 in_params.rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid;
3175
3176 if (qp->srq) {
3177 /* QP is associated with SRQ instead of RQ */
3178 in_params.srq_id = qp->srq->srq_id;
3179 in_params.use_srq = true;
3180 QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n",
3181 in_params.srq_id, in_params.use_srq);
3182 } else {
3183 in_params.srq_id = 0;
3184 in_params.use_srq = false;
3185 }
3186
3187 n_sq_entries = attrs->cap.max_send_wr;
3188 n_sq_entries = min_t(u32, n_sq_entries, qattr->max_wqe);
3189 n_sq_entries = max_t(u32, n_sq_entries, 1);
3190 n_sq_elems = n_sq_entries * QLNXR_MAX_SQE_ELEMENTS_PER_SQE;
3191
3192 n_rq_elems = qp->rq.max_wr * QLNXR_MAX_RQE_ELEMENTS_PER_RQE;
3193
3194 if (QLNX_IS_ROCE(dev)) {
3195 rc = qlnxr_roce_create_kernel_qp(dev, qp, &in_params,
3196 n_sq_elems, n_rq_elems);
3197 } else {
3198 rc = qlnxr_iwarp_create_kernel_qp(dev, qp, &in_params,
3199 n_sq_elems, n_rq_elems);
3200 }
3201
3202 if (rc)
3203 qlnxr_cleanup_kernel(dev, qp);
3204
3205 QL_DPRINT12(ha, "exit [%d]\n", rc);
3206 return rc;
3207 }
3208
3209 struct ib_qp *
3210 qlnxr_create_qp(struct ib_pd *ibpd,
3211 struct ib_qp_init_attr *attrs,
3212 struct ib_udata *udata)
3213 {
3214 struct qlnxr_dev *dev = get_qlnxr_dev(ibpd->device);
3215 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
3216 struct qlnxr_qp *qp;
3217 int rc = 0;
3218 qlnx_host_t *ha;
3219
3220 ha = dev->ha;
3221
3222 QL_DPRINT12(ha, "enter\n");
3223
3224 rc = qlnxr_check_qp_attrs(ibpd, dev, attrs, udata);
3225 if (rc) {
3226 QL_DPRINT11(ha, "qlnxr_check_qp_attrs failed [%d]\n", rc);
3227 return ERR_PTR(rc);
3228 }
3229
3230 QL_DPRINT12(ha, "called from %s, event_handle=%p,"
3231 " eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
3232 (udata ? "user library" : "kernel"),
3233 attrs->event_handler, pd,
3234 get_qlnxr_cq(attrs->send_cq),
3235 get_qlnxr_cq(attrs->send_cq)->icid,
3236 get_qlnxr_cq(attrs->recv_cq),
3237 get_qlnxr_cq(attrs->recv_cq)->icid);
3238
3239 qp = qlnx_zalloc(sizeof(struct qlnxr_qp));
3240
3241 if (!qp) {
3242 QL_DPRINT11(ha, "kzalloc(qp) failed\n");
3243 return ERR_PTR(-ENOMEM);
3244 }
3245
3246 qlnxr_set_common_qp_params(dev, qp, pd, attrs);
3247
3248 if (attrs->qp_type == IB_QPT_GSI) {
3249 QL_DPRINT11(ha, "calling qlnxr_create_gsi_qp\n");
3250 return qlnxr_create_gsi_qp(dev, attrs, qp);
3251 }
3252
3253 if (udata) {
3254 rc = qlnxr_create_user_qp(dev, qp, ibpd, udata, attrs);
3255
3256 if (rc) {
3257 QL_DPRINT11(ha, "qlnxr_create_user_qp failed\n");
3258 goto err;
3259 }
3260 } else {
3261 rc = qlnxr_create_kernel_qp(dev, qp, ibpd, attrs);
3262
3263 if (rc) {
3264 QL_DPRINT11(ha, "qlnxr_create_kernel_qp failed\n");
3265 goto err;
3266 }
3267 }
3268
3269 qp->ibqp.qp_num = qp->qp_id;
3270
3271 rc = qlnxr_idr_add(dev, qp, qp->qp_id);
3272
3273 if (rc) {
3274 QL_DPRINT11(ha, "qlnxr_idr_add failed\n");
3275 goto err;
3276 }
3277
3278 QL_DPRINT12(ha, "exit [%p]\n", &qp->ibqp);
3279
3280 return &qp->ibqp;
3281 err:
3282 kfree(qp);
3283
3284 QL_DPRINT12(ha, "failed exit\n");
3285 return ERR_PTR(-EFAULT);
3286 }
3287
3288 static enum ib_qp_state
3289 qlnxr_get_ibqp_state(enum ecore_roce_qp_state qp_state)
3290 {
3291 enum ib_qp_state state = IB_QPS_ERR;
3292
3293 switch (qp_state) {
3294 case ECORE_ROCE_QP_STATE_RESET:
3295 state = IB_QPS_RESET;
3296 break;
3297
3298 case ECORE_ROCE_QP_STATE_INIT:
3299 state = IB_QPS_INIT;
3300 break;
3301
3302 case ECORE_ROCE_QP_STATE_RTR:
3303 state = IB_QPS_RTR;
3304 break;
3305
3306 case ECORE_ROCE_QP_STATE_RTS:
3307 state = IB_QPS_RTS;
3308 break;
3309
3310 case ECORE_ROCE_QP_STATE_SQD:
3311 state = IB_QPS_SQD;
3312 break;
3313
3314 case ECORE_ROCE_QP_STATE_ERR:
3315 state = IB_QPS_ERR;
3316 break;
3317
3318 case ECORE_ROCE_QP_STATE_SQE:
3319 state = IB_QPS_SQE;
3320 break;
3321 }
3322 return state;
3323 }
3324
3325 static enum ecore_roce_qp_state
3326 qlnxr_get_state_from_ibqp( enum ib_qp_state qp_state)
3327 {
3328 enum ecore_roce_qp_state ecore_qp_state;
3329
3330 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR;
3331
3332 switch (qp_state) {
3333 case IB_QPS_RESET:
3334 ecore_qp_state = ECORE_ROCE_QP_STATE_RESET;
3335 break;
3336
3337 case IB_QPS_INIT:
3338 ecore_qp_state = ECORE_ROCE_QP_STATE_INIT;
3339 break;
3340
3341 case IB_QPS_RTR:
3342 ecore_qp_state = ECORE_ROCE_QP_STATE_RTR;
3343 break;
3344
3345 case IB_QPS_RTS:
3346 ecore_qp_state = ECORE_ROCE_QP_STATE_RTS;
3347 break;
3348
3349 case IB_QPS_SQD:
3350 ecore_qp_state = ECORE_ROCE_QP_STATE_SQD;
3351 break;
3352
3353 case IB_QPS_ERR:
3354 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR;
3355 break;
3356
3357 default:
3358 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR;
3359 break;
3360 }
3361
3362 return (ecore_qp_state);
3363 }
3364
3365 static void
3366 qlnxr_reset_qp_hwq_info(struct qlnxr_qp_hwq_info *qph)
3367 {
3368 ecore_chain_reset(&qph->pbl);
3369 qph->prod = qph->cons = 0;
3370 qph->wqe_cons = 0;
3371 qph->db_data.data.value = cpu_to_le16(0);
3372
3373 return;
3374 }
3375
3376 static int
3377 qlnxr_update_qp_state(struct qlnxr_dev *dev,
3378 struct qlnxr_qp *qp,
3379 enum ecore_roce_qp_state new_state)
3380 {
3381 int status = 0;
3382 uint32_t reg_addr;
3383 struct ecore_dev *cdev;
3384 qlnx_host_t *ha;
3385
3386 ha = dev->ha;
3387 cdev = &ha->cdev;
3388
3389 QL_DPRINT12(ha, "enter qp = %p new_state = 0x%x qp->state = 0x%x\n",
3390 qp, new_state, qp->state);
3391
3392 if (new_state == qp->state) {
3393 return 0;
3394 }
3395
3396 switch (qp->state) {
3397 case ECORE_ROCE_QP_STATE_RESET:
3398 switch (new_state) {
3399 case ECORE_ROCE_QP_STATE_INIT:
3400 qp->prev_wqe_size = 0;
3401 qlnxr_reset_qp_hwq_info(&qp->sq);
3402 if (!(qp->srq))
3403 qlnxr_reset_qp_hwq_info(&qp->rq);
3404 break;
3405 default:
3406 status = -EINVAL;
3407 break;
3408 };
3409 break;
3410 case ECORE_ROCE_QP_STATE_INIT:
3411 /* INIT->XXX */
3412 switch (new_state) {
3413 case ECORE_ROCE_QP_STATE_RTR:
3414 /* Update doorbell (in case post_recv was done before move to RTR) */
3415 if (qp->srq)
3416 break;
3417 wmb();
3418 //writel(qp->rq.db_data.raw, qp->rq.db);
3419 //if (QLNX_IS_IWARP(dev))
3420 // writel(qp->rq.iwarp_db2_data.raw,
3421 // qp->rq.iwarp_db2);
3422
3423 reg_addr = (uint32_t)((uint8_t *)qp->rq.db -
3424 (uint8_t *)cdev->doorbells);
3425
3426 bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw);
3427 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ);
3428
3429 if (QLNX_IS_IWARP(dev)) {
3430 reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 -
3431 (uint8_t *)cdev->doorbells);
3432 bus_write_4(ha->pci_dbells, reg_addr,\
3433 qp->rq.iwarp_db2_data.raw);
3434 bus_barrier(ha->pci_dbells, 0, 0,\
3435 BUS_SPACE_BARRIER_READ);
3436 }
3437
3438
3439 mmiowb();
3440 break;
3441 case ECORE_ROCE_QP_STATE_ERR:
3442 /* TBD:flush qps... */
3443 break;
3444 default:
3445 /* invalid state change. */
3446 status = -EINVAL;
3447 break;
3448 };
3449 break;
3450 case ECORE_ROCE_QP_STATE_RTR:
3451 /* RTR->XXX */
3452 switch (new_state) {
3453 case ECORE_ROCE_QP_STATE_RTS:
3454 break;
3455 case ECORE_ROCE_QP_STATE_ERR:
3456 break;
3457 default:
3458 /* invalid state change. */
3459 status = -EINVAL;
3460 break;
3461 };
3462 break;
3463 case ECORE_ROCE_QP_STATE_RTS:
3464 /* RTS->XXX */
3465 switch (new_state) {
3466 case ECORE_ROCE_QP_STATE_SQD:
3467 break;
3468 case ECORE_ROCE_QP_STATE_ERR:
3469 break;
3470 default:
3471 /* invalid state change. */
3472 status = -EINVAL;
3473 break;
3474 };
3475 break;
3476 case ECORE_ROCE_QP_STATE_SQD:
3477 /* SQD->XXX */
3478 switch (new_state) {
3479 case ECORE_ROCE_QP_STATE_RTS:
3480 case ECORE_ROCE_QP_STATE_ERR:
3481 break;
3482 default:
3483 /* invalid state change. */
3484 status = -EINVAL;
3485 break;
3486 };
3487 break;
3488 case ECORE_ROCE_QP_STATE_ERR:
3489 /* ERR->XXX */
3490 switch (new_state) {
3491 case ECORE_ROCE_QP_STATE_RESET:
3492 if ((qp->rq.prod != qp->rq.cons) ||
3493 (qp->sq.prod != qp->sq.cons)) {
3494 QL_DPRINT11(ha,
3495 "Error->Reset with rq/sq "
3496 "not empty rq.prod=0x%x rq.cons=0x%x"
3497 " sq.prod=0x%x sq.cons=0x%x\n",
3498 qp->rq.prod, qp->rq.cons,
3499 qp->sq.prod, qp->sq.cons);
3500 status = -EINVAL;
3501 }
3502 break;
3503 default:
3504 status = -EINVAL;
3505 break;
3506 };
3507 break;
3508 default:
3509 status = -EINVAL;
3510 break;
3511 };
3512
3513 QL_DPRINT12(ha, "exit\n");
3514 return status;
3515 }
3516
3517 int
3518 qlnxr_modify_qp(struct ib_qp *ibqp,
3519 struct ib_qp_attr *attr,
3520 int attr_mask,
3521 struct ib_udata *udata)
3522 {
3523 int rc = 0;
3524 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
3525 struct qlnxr_dev *dev = get_qlnxr_dev(&qp->dev->ibdev);
3526 struct ecore_rdma_modify_qp_in_params qp_params = { 0 };
3527 enum ib_qp_state old_qp_state, new_qp_state;
3528 struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx);
3529 qlnx_host_t *ha;
3530
3531 ha = dev->ha;
3532
3533 QL_DPRINT12(ha,
3534 "enter qp = %p attr_mask = 0x%x, state = %d udata = %p\n",
3535 qp, attr_mask, attr->qp_state, udata);
3536
3537 old_qp_state = qlnxr_get_ibqp_state(qp->state);
3538 if (attr_mask & IB_QP_STATE)
3539 new_qp_state = attr->qp_state;
3540 else
3541 new_qp_state = old_qp_state;
3542
3543 if (QLNX_IS_ROCE(dev)) {
3544 if (!ib_modify_qp_is_ok(old_qp_state,
3545 new_qp_state,
3546 ibqp->qp_type,
3547 attr_mask )) {
3548 QL_DPRINT12(ha,
3549 "invalid attribute mask=0x%x"
3550 " specified for qpn=0x%x of type=0x%x \n"
3551 " old_qp_state=0x%x, new_qp_state=0x%x\n",
3552 attr_mask, qp->qp_id, ibqp->qp_type,
3553 old_qp_state, new_qp_state);
3554 rc = -EINVAL;
3555 goto err;
3556 }
3557 }
3558 /* translate the masks... */
3559 if (attr_mask & IB_QP_STATE) {
3560 SET_FIELD(qp_params.modify_flags,
3561 ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
3562 qp_params.new_state = qlnxr_get_state_from_ibqp(attr->qp_state);
3563 }
3564
3565 // TBD consider changing ecore to be a flag as well...
3566 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
3567 qp_params.sqd_async = true;
3568
3569 if (attr_mask & IB_QP_PKEY_INDEX) {
3570 SET_FIELD(qp_params.modify_flags,
3571 ECORE_ROCE_MODIFY_QP_VALID_PKEY,
3572 1);
3573 if (attr->pkey_index >= QLNXR_ROCE_PKEY_TABLE_LEN) {
3574 rc = -EINVAL;
3575 goto err;
3576 }
3577
3578 qp_params.pkey = QLNXR_ROCE_PKEY_DEFAULT;
3579 }
3580
3581 if (attr_mask & IB_QP_QKEY) {
3582 qp->qkey = attr->qkey;
3583 }
3584
3585 /* tbd consider splitting in ecore.. */
3586 if (attr_mask & IB_QP_ACCESS_FLAGS) {
3587 SET_FIELD(qp_params.modify_flags,
3588 ECORE_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
3589 qp_params.incoming_rdma_read_en =
3590 attr->qp_access_flags & IB_ACCESS_REMOTE_READ;
3591 qp_params.incoming_rdma_write_en =
3592 attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE;
3593 qp_params.incoming_atomic_en =
3594 attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC;
3595 }
3596
3597 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
3598 if (attr_mask & IB_QP_PATH_MTU) {
3599 if (attr->path_mtu < IB_MTU_256 ||
3600 attr->path_mtu > IB_MTU_4096) {
3601 QL_DPRINT12(ha,
3602 "Only MTU sizes of 256, 512, 1024,"
3603 " 2048 and 4096 are supported "
3604 " attr->path_mtu = [%d]\n",
3605 attr->path_mtu);
3606
3607 rc = -EINVAL;
3608 goto err;
3609 }
3610 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
3611 ib_mtu_enum_to_int(
3612 iboe_get_mtu(dev->ha->ifp->if_mtu)));
3613 }
3614
3615 if (qp->mtu == 0) {
3616 qp->mtu = ib_mtu_enum_to_int(
3617 iboe_get_mtu(dev->ha->ifp->if_mtu));
3618 QL_DPRINT12(ha, "fixing zetoed MTU to qp->mtu = %d\n",
3619 qp->mtu);
3620 }
3621
3622 SET_FIELD(qp_params.modify_flags,
3623 ECORE_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR,
3624 1);
3625
3626 qp_params.traffic_class_tos = attr->ah_attr.grh.traffic_class;
3627 qp_params.flow_label = attr->ah_attr.grh.flow_label;
3628 qp_params.hop_limit_ttl = attr->ah_attr.grh.hop_limit;
3629
3630 qp->sgid_idx = attr->ah_attr.grh.sgid_index;
3631
3632 get_gid_info(ibqp, attr, attr_mask, dev, qp, &qp_params);
3633
3634 rc = qlnxr_get_dmac(dev, &attr->ah_attr, qp_params.remote_mac_addr);
3635 if (rc)
3636 return rc;
3637
3638 qp_params.use_local_mac = true;
3639 memcpy(qp_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN);
3640
3641 QL_DPRINT12(ha, "dgid=0x%x:0x%x:0x%x:0x%x\n",
3642 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
3643 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
3644 QL_DPRINT12(ha, "sgid=0x%x:0x%x:0x%x:0x%x\n",
3645 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
3646 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
3647 QL_DPRINT12(ha,
3648 "remote_mac=[0x%x:0x%x:0x%x:0x%x:0x%x:0x%x]\n",
3649 qp_params.remote_mac_addr[0],
3650 qp_params.remote_mac_addr[1],
3651 qp_params.remote_mac_addr[2],
3652 qp_params.remote_mac_addr[3],
3653 qp_params.remote_mac_addr[4],
3654 qp_params.remote_mac_addr[5]);
3655
3656 qp_params.mtu = qp->mtu;
3657 }
3658
3659 if (qp_params.mtu == 0) {
3660 /* stay with current MTU */
3661 if (qp->mtu) {
3662 qp_params.mtu = qp->mtu;
3663 } else {
3664 qp_params.mtu = ib_mtu_enum_to_int(
3665 iboe_get_mtu(dev->ha->ifp->if_mtu));
3666 }
3667 }
3668
3669 if (attr_mask & IB_QP_TIMEOUT) {
3670 SET_FIELD(qp_params.modify_flags, \
3671 ECORE_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
3672
3673 qp_params.ack_timeout = attr->timeout;
3674 if (attr->timeout) {
3675 u32 temp;
3676
3677 /* 12.7.34 LOCAL ACK TIMEOUT
3678 * Value representing the transport (ACK) timeout for
3679 * use by the remote, expressed as (4.096 μS*2Local ACK
3680 * Timeout)
3681 */
3682 /* We use 1UL since the temporal value may be overflow
3683 * 32 bits
3684 */
3685 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;
3686 qp_params.ack_timeout = temp; /* FW requires [msec] */
3687 }
3688 else
3689 qp_params.ack_timeout = 0; /* infinite */
3690 }
3691 if (attr_mask & IB_QP_RETRY_CNT) {
3692 SET_FIELD(qp_params.modify_flags,\
3693 ECORE_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
3694 qp_params.retry_cnt = attr->retry_cnt;
3695 }
3696
3697 if (attr_mask & IB_QP_RNR_RETRY) {
3698 SET_FIELD(qp_params.modify_flags,
3699 ECORE_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT,
3700 1);
3701 qp_params.rnr_retry_cnt = attr->rnr_retry;
3702 }
3703
3704 if (attr_mask & IB_QP_RQ_PSN) {
3705 SET_FIELD(qp_params.modify_flags,
3706 ECORE_ROCE_MODIFY_QP_VALID_RQ_PSN,
3707 1);
3708 qp_params.rq_psn = attr->rq_psn;
3709 qp->rq_psn = attr->rq_psn;
3710 }
3711
3712 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
3713 if (attr->max_rd_atomic > qattr->max_qp_req_rd_atomic_resc) {
3714 rc = -EINVAL;
3715 QL_DPRINT12(ha,
3716 "unsupported max_rd_atomic=%d, supported=%d\n",
3717 attr->max_rd_atomic,
3718 qattr->max_qp_req_rd_atomic_resc);
3719 goto err;
3720 }
3721
3722 SET_FIELD(qp_params.modify_flags,
3723 ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ,
3724 1);
3725 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
3726 }
3727
3728 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
3729 SET_FIELD(qp_params.modify_flags,
3730 ECORE_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER,
3731 1);
3732 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
3733 }
3734
3735 if (attr_mask & IB_QP_SQ_PSN) {
3736 SET_FIELD(qp_params.modify_flags,
3737 ECORE_ROCE_MODIFY_QP_VALID_SQ_PSN,
3738 1);
3739 qp_params.sq_psn = attr->sq_psn;
3740 qp->sq_psn = attr->sq_psn;
3741 }
3742
3743 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
3744 if (attr->max_dest_rd_atomic >
3745 qattr->max_qp_resp_rd_atomic_resc) {
3746 QL_DPRINT12(ha,
3747 "unsupported max_dest_rd_atomic=%d, "
3748 "supported=%d\n",
3749 attr->max_dest_rd_atomic,
3750 qattr->max_qp_resp_rd_atomic_resc);
3751
3752 rc = -EINVAL;
3753 goto err;
3754 }
3755
3756 SET_FIELD(qp_params.modify_flags,
3757 ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP,
3758 1);
3759 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
3760 }
3761
3762 if (attr_mask & IB_QP_DEST_QPN) {
3763 SET_FIELD(qp_params.modify_flags,
3764 ECORE_ROCE_MODIFY_QP_VALID_DEST_QP,
3765 1);
3766
3767 qp_params.dest_qp = attr->dest_qp_num;
3768 qp->dest_qp_num = attr->dest_qp_num;
3769 }
3770
3771 /*
3772 * Update the QP state before the actual ramrod to prevent a race with
3773 * fast path. Modifying the QP state to error will cause the device to
3774 * flush the CQEs and while polling the flushed CQEs will considered as
3775 * a potential issue if the QP isn't in error state.
3776 */
3777 if ((attr_mask & IB_QP_STATE) && (qp->qp_type != IB_QPT_GSI) &&
3778 (!udata) && (qp_params.new_state == ECORE_ROCE_QP_STATE_ERR))
3779 qp->state = ECORE_ROCE_QP_STATE_ERR;
3780
3781 if (qp->qp_type != IB_QPT_GSI)
3782 rc = ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params);
3783
3784 if (attr_mask & IB_QP_STATE) {
3785 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
3786 rc = qlnxr_update_qp_state(dev, qp, qp_params.new_state);
3787 qp->state = qp_params.new_state;
3788 }
3789
3790 err:
3791 QL_DPRINT12(ha, "exit\n");
3792 return rc;
3793 }
3794
3795 static int
3796 qlnxr_to_ib_qp_acc_flags(struct ecore_rdma_query_qp_out_params *params)
3797 {
3798 int ib_qp_acc_flags = 0;
3799
3800 if (params->incoming_rdma_write_en)
3801 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
3802 if (params->incoming_rdma_read_en)
3803 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
3804 if (params->incoming_atomic_en)
3805 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
3806 if (true) /* FIXME -> local write ?? */
3807 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
3808
3809 return ib_qp_acc_flags;
3810 }
3811
3812 static enum ib_mtu
3813 qlnxr_mtu_int_to_enum(u16 mtu)
3814 {
3815 enum ib_mtu ib_mtu_size;
3816
3817 switch (mtu) {
3818 case 256:
3819 ib_mtu_size = IB_MTU_256;
3820 break;
3821
3822 case 512:
3823 ib_mtu_size = IB_MTU_512;
3824 break;
3825
3826 case 1024:
3827 ib_mtu_size = IB_MTU_1024;
3828 break;
3829
3830 case 2048:
3831 ib_mtu_size = IB_MTU_2048;
3832 break;
3833
3834 case 4096:
3835 ib_mtu_size = IB_MTU_4096;
3836 break;
3837
3838 default:
3839 ib_mtu_size = IB_MTU_1024;
3840 break;
3841 }
3842 return (ib_mtu_size);
3843 }
3844
3845 int
3846 qlnxr_query_qp(struct ib_qp *ibqp,
3847 struct ib_qp_attr *qp_attr,
3848 int attr_mask,
3849 struct ib_qp_init_attr *qp_init_attr)
3850 {
3851 int rc = 0;
3852 struct ecore_rdma_query_qp_out_params params;
3853 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
3854 struct qlnxr_dev *dev = qp->dev;
3855 qlnx_host_t *ha;
3856
3857 ha = dev->ha;
3858
3859 QL_DPRINT12(ha, "enter\n");
3860
3861 memset(¶ms, 0, sizeof(params));
3862
3863 rc = ecore_rdma_query_qp(dev->rdma_ctx, qp->ecore_qp, ¶ms);
3864 if (rc)
3865 goto err;
3866
3867 memset(qp_attr, 0, sizeof(*qp_attr));
3868 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
3869
3870 qp_attr->qp_state = qlnxr_get_ibqp_state(params.state);
3871 qp_attr->cur_qp_state = qlnxr_get_ibqp_state(params.state);
3872
3873 /* In some cases in iWARP qelr will ask for the state only */
3874 if (QLNX_IS_IWARP(dev) && (attr_mask == IB_QP_STATE)) {
3875 QL_DPRINT11(ha, "only state requested\n");
3876 return 0;
3877 }
3878
3879 qp_attr->path_mtu = qlnxr_mtu_int_to_enum(params.mtu);
3880 qp_attr->path_mig_state = IB_MIG_MIGRATED;
3881 qp_attr->rq_psn = params.rq_psn;
3882 qp_attr->sq_psn = params.sq_psn;
3883 qp_attr->dest_qp_num = params.dest_qp;
3884
3885 qp_attr->qp_access_flags = qlnxr_to_ib_qp_acc_flags(¶ms);
3886
3887 QL_DPRINT12(ha, "qp_state = 0x%x cur_qp_state = 0x%x "
3888 "path_mtu = %d qp_access_flags = 0x%x\n",
3889 qp_attr->qp_state, qp_attr->cur_qp_state, qp_attr->path_mtu,
3890 qp_attr->qp_access_flags);
3891
3892 qp_attr->cap.max_send_wr = qp->sq.max_wr;
3893 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
3894 qp_attr->cap.max_send_sge = qp->sq.max_sges;
3895 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
3896 qp_attr->cap.max_inline_data = qp->max_inline_data;
3897 qp_init_attr->cap = qp_attr->cap;
3898
3899 memcpy(&qp_attr->ah_attr.grh.dgid.raw[0], ¶ms.dgid.bytes[0],
3900 sizeof(qp_attr->ah_attr.grh.dgid.raw));
3901
3902 qp_attr->ah_attr.grh.flow_label = params.flow_label;
3903 qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx;
3904 qp_attr->ah_attr.grh.hop_limit = params.hop_limit_ttl;
3905 qp_attr->ah_attr.grh.traffic_class = params.traffic_class_tos;
3906
3907 qp_attr->ah_attr.ah_flags = IB_AH_GRH;
3908 qp_attr->ah_attr.port_num = 1; /* FIXME -> check this */
3909 qp_attr->ah_attr.sl = 0;/* FIXME -> check this */
3910 qp_attr->timeout = params.timeout;
3911 qp_attr->rnr_retry = params.rnr_retry;
3912 qp_attr->retry_cnt = params.retry_cnt;
3913 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
3914 qp_attr->pkey_index = params.pkey_index;
3915 qp_attr->port_num = 1; /* FIXME -> check this */
3916 qp_attr->ah_attr.src_path_bits = 0;
3917 qp_attr->ah_attr.static_rate = 0;
3918 qp_attr->alt_pkey_index = 0;
3919 qp_attr->alt_port_num = 0;
3920 qp_attr->alt_timeout = 0;
3921 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
3922
3923 qp_attr->sq_draining = (params.state == ECORE_ROCE_QP_STATE_SQD) ? 1 : 0;
3924 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
3925 qp_attr->max_rd_atomic = params.max_rd_atomic;
3926 qp_attr->en_sqd_async_notify = (params.sqd_async)? 1 : 0;
3927
3928 QL_DPRINT12(ha, "max_inline_data=%d\n",
3929 qp_attr->cap.max_inline_data);
3930
3931 err:
3932 QL_DPRINT12(ha, "exit\n");
3933 return rc;
3934 }
3935
3936 static void
3937 qlnxr_cleanup_user(struct qlnxr_dev *dev, struct qlnxr_qp *qp)
3938 {
3939 qlnx_host_t *ha;
3940
3941 ha = dev->ha;
3942
3943 QL_DPRINT12(ha, "enter\n");
3944
3945 if (qp->usq.umem)
3946 ib_umem_release(qp->usq.umem);
3947
3948 qp->usq.umem = NULL;
3949
3950 if (qp->urq.umem)
3951 ib_umem_release(qp->urq.umem);
3952
3953 qp->urq.umem = NULL;
3954
3955 QL_DPRINT12(ha, "exit\n");
3956 return;
3957 }
3958
3959 static void
3960 qlnxr_cleanup_kernel(struct qlnxr_dev *dev, struct qlnxr_qp *qp)
3961 {
3962 qlnx_host_t *ha;
3963
3964 ha = dev->ha;
3965
3966 QL_DPRINT12(ha, "enter\n");
3967
3968 if (qlnxr_qp_has_sq(qp)) {
3969 QL_DPRINT12(ha, "freeing SQ\n");
3970 ha->qlnxr_debug = 1;
3971 // ecore_chain_free(dev->cdev, &qp->sq.pbl);
3972 ha->qlnxr_debug = 0;
3973 kfree(qp->wqe_wr_id);
3974 }
3975
3976 if (qlnxr_qp_has_rq(qp)) {
3977 QL_DPRINT12(ha, "freeing RQ\n");
3978 ha->qlnxr_debug = 1;
3979 // ecore_chain_free(dev->cdev, &qp->rq.pbl);
3980 ha->qlnxr_debug = 0;
3981 kfree(qp->rqe_wr_id);
3982 }
3983
3984 QL_DPRINT12(ha, "exit\n");
3985 return;
3986 }
3987
3988 int
3989 qlnxr_free_qp_resources(struct qlnxr_dev *dev,
3990 struct qlnxr_qp *qp)
3991 {
3992 int rc = 0;
3993 qlnx_host_t *ha;
3994 struct ecore_rdma_destroy_qp_out_params d_out_params;
3995
3996 ha = dev->ha;
3997
3998 QL_DPRINT12(ha, "enter\n");
3999
4000 #if 0
4001 if (qp->qp_type != IB_QPT_GSI) {
4002 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp,
4003 &d_out_params);
4004 if (rc)
4005 return rc;
4006 }
4007
4008 if (qp->ibqp.uobject && qp->ibqp.uobject->context)
4009 qlnxr_cleanup_user(dev, qp);
4010 else
4011 qlnxr_cleanup_kernel(dev, qp);
4012 #endif
4013
4014 if (qp->ibqp.uobject && qp->ibqp.uobject->context)
4015 qlnxr_cleanup_user(dev, qp);
4016 else
4017 qlnxr_cleanup_kernel(dev, qp);
4018
4019 if (qp->qp_type != IB_QPT_GSI) {
4020 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp,
4021 &d_out_params);
4022 if (rc)
4023 return rc;
4024 }
4025
4026 QL_DPRINT12(ha, "exit\n");
4027 return 0;
4028 }
4029
4030 int
4031 qlnxr_destroy_qp(struct ib_qp *ibqp)
4032 {
4033 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
4034 struct qlnxr_dev *dev = qp->dev;
4035 int rc = 0;
4036 struct ib_qp_attr attr;
4037 int attr_mask = 0;
4038 qlnx_host_t *ha;
4039
4040 ha = dev->ha;
4041
4042 QL_DPRINT12(ha, "enter qp = %p, qp_type=%d\n", qp, qp->qp_type);
4043
4044 qp->destroyed = 1;
4045
4046 if (QLNX_IS_ROCE(dev) && (qp->state != (ECORE_ROCE_QP_STATE_RESET |
4047 ECORE_ROCE_QP_STATE_ERR |
4048 ECORE_ROCE_QP_STATE_INIT))) {
4049 attr.qp_state = IB_QPS_ERR;
4050 attr_mask |= IB_QP_STATE;
4051
4052 /* change the QP state to ERROR */
4053 qlnxr_modify_qp(ibqp, &attr, attr_mask, NULL);
4054 }
4055
4056 if (qp->qp_type == IB_QPT_GSI)
4057 qlnxr_destroy_gsi_qp(dev);
4058
4059 qp->sig = ~qp->sig;
4060
4061 qlnxr_free_qp_resources(dev, qp);
4062
4063 if (atomic_dec_and_test(&qp->refcnt)) {
4064 /* TODO: only for iWARP? */
4065 qlnxr_idr_remove(dev, qp->qp_id);
4066 kfree(qp);
4067 }
4068
4069 QL_DPRINT12(ha, "exit\n");
4070 return rc;
4071 }
4072
4073 static inline int
4074 qlnxr_wq_is_full(struct qlnxr_qp_hwq_info *wq)
4075 {
4076 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
4077 }
4078
4079 static int
4080 sge_data_len(struct ib_sge *sg_list, int num_sge)
4081 {
4082 int i, len = 0;
4083 for (i = 0; i < num_sge; i++)
4084 len += sg_list[i].length;
4085 return len;
4086 }
4087
4088 static void
4089 swap_wqe_data64(u64 *p)
4090 {
4091 int i;
4092
4093 for (i = 0; i < QLNXR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
4094 *p = cpu_to_be64(cpu_to_le64(*p));
4095 }
4096
4097 static u32
4098 qlnxr_prepare_sq_inline_data(struct qlnxr_dev *dev,
4099 struct qlnxr_qp *qp,
4100 u8 *wqe_size,
4101 const struct ib_send_wr *wr,
4102 const struct ib_send_wr **bad_wr,
4103 u8 *bits,
4104 u8 bit)
4105 {
4106 int i, seg_siz;
4107 char *seg_prt, *wqe;
4108 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
4109 qlnx_host_t *ha;
4110
4111 ha = dev->ha;
4112
4113 QL_DPRINT12(ha, "enter[%d]\n", data_size);
4114
4115 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
4116 QL_DPRINT12(ha,
4117 "Too much inline data in WR:[%d, %d]\n",
4118 data_size, ROCE_REQ_MAX_INLINE_DATA_SIZE);
4119 *bad_wr = wr;
4120 return 0;
4121 }
4122
4123 if (!data_size)
4124 return data_size;
4125
4126 /* set the bit */
4127 *bits |= bit;
4128
4129 seg_prt = wqe = NULL;
4130 seg_siz = 0;
4131
4132 /* copy data inline */
4133 for (i = 0; i < wr->num_sge; i++) {
4134 u32 len = wr->sg_list[i].length;
4135 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
4136
4137 while (len > 0) {
4138 u32 cur;
4139
4140 /* new segment required */
4141 if (!seg_siz) {
4142 wqe = (char *)ecore_chain_produce(&qp->sq.pbl);
4143 seg_prt = wqe;
4144 seg_siz = sizeof(struct rdma_sq_common_wqe);
4145 (*wqe_size)++;
4146 }
4147
4148 /* calculate currently allowed length */
4149 cur = MIN(len, seg_siz);
4150
4151 memcpy(seg_prt, src, cur);
4152
4153 /* update segment variables */
4154 seg_prt += cur;
4155 seg_siz -= cur;
4156 /* update sge variables */
4157 src += cur;
4158 len -= cur;
4159
4160 /* swap fully-completed segments */
4161 if (!seg_siz)
4162 swap_wqe_data64((u64 *)wqe);
4163 }
4164 }
4165
4166 /* swap last not completed segment */
4167 if (seg_siz)
4168 swap_wqe_data64((u64 *)wqe);
4169
4170 QL_DPRINT12(ha, "exit\n");
4171 return data_size;
4172 }
4173
4174 static u32
4175 qlnxr_prepare_sq_sges(struct qlnxr_dev *dev, struct qlnxr_qp *qp,
4176 u8 *wqe_size, const struct ib_send_wr *wr)
4177 {
4178 int i;
4179 u32 data_size = 0;
4180 qlnx_host_t *ha;
4181
4182 ha = dev->ha;
4183
4184 QL_DPRINT12(ha, "enter wr->num_sge = %d \n", wr->num_sge);
4185
4186 for (i = 0; i < wr->num_sge; i++) {
4187 struct rdma_sq_sge *sge = ecore_chain_produce(&qp->sq.pbl);
4188
4189 TYPEPTR_ADDR_SET(sge, addr, wr->sg_list[i].addr);
4190 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
4191 sge->length = cpu_to_le32(wr->sg_list[i].length);
4192 data_size += wr->sg_list[i].length;
4193 }
4194
4195 if (wqe_size)
4196 *wqe_size += wr->num_sge;
4197
4198 QL_DPRINT12(ha, "exit data_size = %d\n", data_size);
4199 return data_size;
4200 }
4201
4202 static u32
4203 qlnxr_prepare_sq_rdma_data(struct qlnxr_dev *dev,
4204 struct qlnxr_qp *qp,
4205 struct rdma_sq_rdma_wqe_1st *rwqe,
4206 struct rdma_sq_rdma_wqe_2nd *rwqe2,
4207 const struct ib_send_wr *wr,
4208 const struct ib_send_wr **bad_wr)
4209 {
4210 qlnx_host_t *ha;
4211 u32 ret = 0;
4212
4213 ha = dev->ha;
4214
4215 QL_DPRINT12(ha, "enter\n");
4216
4217 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
4218 TYPEPTR_ADDR_SET(rwqe2, remote_va, rdma_wr(wr)->remote_addr);
4219
4220 if (wr->send_flags & IB_SEND_INLINE) {
4221 u8 flags = 0;
4222 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
4223 return qlnxr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size,
4224 wr, bad_wr, &rwqe->flags, flags);
4225 }
4226
4227 ret = qlnxr_prepare_sq_sges(dev, qp, &rwqe->wqe_size, wr);
4228
4229 QL_DPRINT12(ha, "exit ret = 0x%x\n", ret);
4230
4231 return (ret);
4232 }
4233
4234 static u32
4235 qlnxr_prepare_sq_send_data(struct qlnxr_dev *dev,
4236 struct qlnxr_qp *qp,
4237 struct rdma_sq_send_wqe *swqe,
4238 struct rdma_sq_send_wqe *swqe2,
4239 const struct ib_send_wr *wr,
4240 const struct ib_send_wr **bad_wr)
4241 {
4242 qlnx_host_t *ha;
4243 u32 ret = 0;
4244
4245 ha = dev->ha;
4246
4247 QL_DPRINT12(ha, "enter\n");
4248
4249 memset(swqe2, 0, sizeof(*swqe2));
4250
4251 if (wr->send_flags & IB_SEND_INLINE) {
4252 u8 flags = 0;
4253 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
4254 return qlnxr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size,
4255 wr, bad_wr, &swqe->flags, flags);
4256 }
4257
4258 ret = qlnxr_prepare_sq_sges(dev, qp, &swqe->wqe_size, wr);
4259
4260 QL_DPRINT12(ha, "exit ret = 0x%x\n", ret);
4261
4262 return (ret);
4263 }
4264
4265 static void
4266 qlnx_handle_completed_mrs(struct qlnxr_dev *dev, struct mr_info *info)
4267 {
4268 qlnx_host_t *ha;
4269
4270 ha = dev->ha;
4271
4272 int work = info->completed - info->completed_handled - 1;
4273
4274 QL_DPRINT12(ha, "enter [%d]\n", work);
4275
4276 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
4277 struct qlnxr_pbl *pbl;
4278
4279 /* Free all the page list that are possible to be freed
4280 * (all the ones that were invalidated), under the assumption
4281 * that if an FMR was completed successfully that means that
4282 * if there was an invalidate operation before it also ended
4283 */
4284 pbl = list_first_entry(&info->inuse_pbl_list,
4285 struct qlnxr_pbl,
4286 list_entry);
4287 list_del(&pbl->list_entry);
4288 list_add_tail(&pbl->list_entry, &info->free_pbl_list);
4289 info->completed_handled++;
4290 }
4291
4292 QL_DPRINT12(ha, "exit\n");
4293 return;
4294 }
4295
4296 #if __FreeBSD_version >= 1102000
4297
4298 static int qlnxr_prepare_reg(struct qlnxr_qp *qp,
4299 struct rdma_sq_fmr_wqe_1st *fwqe1,
4300 const struct ib_reg_wr *wr)
4301 {
4302 struct qlnxr_mr *mr = get_qlnxr_mr(wr->mr);
4303 struct rdma_sq_fmr_wqe_2nd *fwqe2;
4304
4305 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)ecore_chain_produce(&qp->sq.pbl);
4306 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
4307 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
4308 fwqe1->l_key = wr->key;
4309
4310 fwqe2->access_ctrl = 0;
4311
4312 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
4313 !!(wr->access & IB_ACCESS_REMOTE_READ));
4314 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
4315 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
4316 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
4317 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
4318 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
4319 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
4320 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
4321 fwqe2->fmr_ctrl = 0;
4322
4323 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
4324 ilog2(mr->ibmr.page_size) - 12);
4325
4326 fwqe2->length_hi = 0; /* TODO - figure out why length is only 32bit.. */
4327 fwqe2->length_lo = mr->ibmr.length;
4328 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
4329 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
4330
4331 qp->wqe_wr_id[qp->sq.prod].mr = mr;
4332
4333 return 0;
4334 }
4335
4336 #else
4337
4338 static void
4339 build_frmr_pbes(struct qlnxr_dev *dev, const struct ib_send_wr *wr,
4340 struct mr_info *info)
4341 {
4342 int i;
4343 u64 buf_addr = 0;
4344 int num_pbes, total_num_pbes = 0;
4345 struct regpair *pbe;
4346 struct qlnxr_pbl *pbl_tbl = info->pbl_table;
4347 struct qlnxr_pbl_info *pbl_info = &info->pbl_info;
4348 qlnx_host_t *ha;
4349
4350 ha = dev->ha;
4351
4352 QL_DPRINT12(ha, "enter\n");
4353
4354 pbe = (struct regpair *)pbl_tbl->va;
4355 num_pbes = 0;
4356
4357 for (i = 0; i < wr->wr.fast_reg.page_list_len; i++) {
4358 buf_addr = wr->wr.fast_reg.page_list->page_list[i];
4359 pbe->lo = cpu_to_le32((u32)buf_addr);
4360 pbe->hi = cpu_to_le32((u32)upper_32_bits(buf_addr));
4361
4362 num_pbes += 1;
4363 pbe++;
4364 total_num_pbes++;
4365
4366 if (total_num_pbes == pbl_info->num_pbes)
4367 return;
4368
4369 /* if the given pbl is full storing the pbes,
4370 * move to next pbl.
4371 */
4372 if (num_pbes ==
4373 (pbl_info->pbl_size / sizeof(u64))) {
4374 pbl_tbl++;
4375 pbe = (struct regpair *)pbl_tbl->va;
4376 num_pbes = 0;
4377 }
4378 }
4379 QL_DPRINT12(ha, "exit\n");
4380
4381 return;
4382 }
4383
4384 static int
4385 qlnxr_prepare_safe_pbl(struct qlnxr_dev *dev, struct mr_info *info)
4386 {
4387 int rc = 0;
4388 qlnx_host_t *ha;
4389
4390 ha = dev->ha;
4391
4392 QL_DPRINT12(ha, "enter\n");
4393
4394 if (info->completed == 0) {
4395 //DP_VERBOSE(dev, QLNXR_MSG_MR, "First FMR\n");
4396 /* first fmr */
4397 return 0;
4398 }
4399
4400 qlnx_handle_completed_mrs(dev, info);
4401
4402 list_add_tail(&info->pbl_table->list_entry, &info->inuse_pbl_list);
4403
4404 if (list_empty(&info->free_pbl_list)) {
4405 info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info,
4406 GFP_ATOMIC);
4407 } else {
4408 info->pbl_table = list_first_entry(&info->free_pbl_list,
4409 struct qlnxr_pbl,
4410 list_entry);
4411 list_del(&info->pbl_table->list_entry);
4412 }
4413
4414 if (!info->pbl_table)
4415 rc = -ENOMEM;
4416
4417 QL_DPRINT12(ha, "exit\n");
4418 return rc;
4419 }
4420
4421 static inline int
4422 qlnxr_prepare_fmr(struct qlnxr_qp *qp,
4423 struct rdma_sq_fmr_wqe_1st *fwqe1,
4424 const struct ib_send_wr *wr)
4425 {
4426 struct qlnxr_dev *dev = qp->dev;
4427 u64 fbo;
4428 struct qlnxr_fast_reg_page_list *frmr_list =
4429 get_qlnxr_frmr_list(wr->wr.fast_reg.page_list);
4430 struct rdma_sq_fmr_wqe *fwqe2 =
4431 (struct rdma_sq_fmr_wqe *)ecore_chain_produce(&qp->sq.pbl);
4432 int rc = 0;
4433 qlnx_host_t *ha;
4434
4435 ha = dev->ha;
4436
4437 QL_DPRINT12(ha, "enter\n");
4438
4439 if (wr->wr.fast_reg.page_list_len == 0)
4440 BUG();
4441
4442 rc = qlnxr_prepare_safe_pbl(dev, &frmr_list->info);
4443 if (rc)
4444 return rc;
4445
4446 fwqe1->addr.hi = upper_32_bits(wr->wr.fast_reg.iova_start);
4447 fwqe1->addr.lo = lower_32_bits(wr->wr.fast_reg.iova_start);
4448 fwqe1->l_key = wr->wr.fast_reg.rkey;
4449
4450 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_REMOTE_READ,
4451 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_READ));
4452 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_REMOTE_WRITE,
4453 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_WRITE));
4454 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_ENABLE_ATOMIC,
4455 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_ATOMIC));
4456 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_LOCAL_READ, 1);
4457 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_LOCAL_WRITE,
4458 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_LOCAL_WRITE));
4459
4460 fwqe2->fmr_ctrl = 0;
4461
4462 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
4463 ilog2(1 << wr->wr.fast_reg.page_shift) - 12);
4464 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_ZERO_BASED, 0);
4465
4466 fwqe2->length_hi = 0; /* Todo - figure this out... why length is only 32bit.. */
4467 fwqe2->length_lo = wr->wr.fast_reg.length;
4468 fwqe2->pbl_addr.hi = upper_32_bits(frmr_list->info.pbl_table->pa);
4469 fwqe2->pbl_addr.lo = lower_32_bits(frmr_list->info.pbl_table->pa);
4470
4471 /* produce another wqe for fwqe3 */
4472 ecore_chain_produce(&qp->sq.pbl);
4473
4474 fbo = wr->wr.fast_reg.iova_start -
4475 (wr->wr.fast_reg.page_list->page_list[0] & PAGE_MASK);
4476
4477 QL_DPRINT12(ha, "wr.fast_reg.iova_start = %p rkey=%x addr=%x:%x"
4478 " length = %x pbl_addr %x:%x\n",
4479 wr->wr.fast_reg.iova_start, wr->wr.fast_reg.rkey,
4480 fwqe1->addr.hi, fwqe1->addr.lo, fwqe2->length_lo,
4481 fwqe2->pbl_addr.hi, fwqe2->pbl_addr.lo);
4482
4483 build_frmr_pbes(dev, wr, &frmr_list->info);
4484
4485 qp->wqe_wr_id[qp->sq.prod].frmr = frmr_list;
4486
4487 QL_DPRINT12(ha, "exit\n");
4488 return 0;
4489 }
4490
4491 #endif /* #if __FreeBSD_version >= 1102000 */
4492
4493 static enum ib_wc_opcode
4494 qlnxr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
4495 {
4496 switch (opcode) {
4497 case IB_WR_RDMA_WRITE:
4498 case IB_WR_RDMA_WRITE_WITH_IMM:
4499 return IB_WC_RDMA_WRITE;
4500 case IB_WR_SEND_WITH_IMM:
4501 case IB_WR_SEND:
4502 case IB_WR_SEND_WITH_INV:
4503 return IB_WC_SEND;
4504 case IB_WR_RDMA_READ:
4505 return IB_WC_RDMA_READ;
4506 case IB_WR_ATOMIC_CMP_AND_SWP:
4507 return IB_WC_COMP_SWAP;
4508 case IB_WR_ATOMIC_FETCH_AND_ADD:
4509 return IB_WC_FETCH_ADD;
4510
4511 #if __FreeBSD_version >= 1102000
4512 case IB_WR_REG_MR:
4513 return IB_WC_REG_MR;
4514 #else
4515 case IB_WR_FAST_REG_MR:
4516 return IB_WC_FAST_REG_MR;
4517 #endif /* #if __FreeBSD_version >= 1102000 */
4518
4519 case IB_WR_LOCAL_INV:
4520 return IB_WC_LOCAL_INV;
4521 default:
4522 return IB_WC_SEND;
4523 }
4524 }
4525 static inline bool
4526 qlnxr_can_post_send(struct qlnxr_qp *qp, const struct ib_send_wr *wr)
4527 {
4528 int wq_is_full, err_wr, pbl_is_full;
4529 struct qlnxr_dev *dev = qp->dev;
4530 qlnx_host_t *ha;
4531
4532 ha = dev->ha;
4533
4534 QL_DPRINT12(ha, "enter[qp, wr] = [%p,%p]\n", qp, wr);
4535
4536 /* prevent SQ overflow and/or processing of a bad WR */
4537 err_wr = wr->num_sge > qp->sq.max_sges;
4538 wq_is_full = qlnxr_wq_is_full(&qp->sq);
4539 pbl_is_full = ecore_chain_get_elem_left_u32(&qp->sq.pbl) <
4540 QLNXR_MAX_SQE_ELEMENTS_PER_SQE;
4541 if (wq_is_full || err_wr || pbl_is_full) {
4542 if (wq_is_full &&
4543 !(qp->err_bitmap & QLNXR_QP_ERR_SQ_FULL)) {
4544 qp->err_bitmap |= QLNXR_QP_ERR_SQ_FULL;
4545
4546 QL_DPRINT12(ha,
4547 "error: WQ is full. Post send on QP failed"
4548 " (this error appears only once) "
4549 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n",
4550 qp, wr, qp->err_bitmap);
4551 }
4552
4553 if (err_wr &&
4554 !(qp->err_bitmap & QLNXR_QP_ERR_BAD_SR)) {
4555 qp->err_bitmap |= QLNXR_QP_ERR_BAD_SR;
4556
4557 QL_DPRINT12(ha,
4558 "error: WQ is bad. Post send on QP failed"
4559 " (this error appears only once) "
4560 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n",
4561 qp, wr, qp->err_bitmap);
4562 }
4563
4564 if (pbl_is_full &&
4565 !(qp->err_bitmap & QLNXR_QP_ERR_SQ_PBL_FULL)) {
4566 qp->err_bitmap |= QLNXR_QP_ERR_SQ_PBL_FULL;
4567
4568 QL_DPRINT12(ha,
4569 "error: WQ PBL is full. Post send on QP failed"
4570 " (this error appears only once) "
4571 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n",
4572 qp, wr, qp->err_bitmap);
4573 }
4574 return false;
4575 }
4576 QL_DPRINT12(ha, "exit[qp, wr] = [%p,%p]\n", qp, wr);
4577 return true;
4578 }
4579
4580 int
4581 qlnxr_post_send(struct ib_qp *ibqp,
4582 const struct ib_send_wr *wr,
4583 const struct ib_send_wr **bad_wr)
4584 {
4585 struct qlnxr_dev *dev = get_qlnxr_dev(ibqp->device);
4586 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
4587 unsigned long flags;
4588 int status = 0, rc = 0;
4589 bool comp;
4590 qlnx_host_t *ha;
4591 uint32_t reg_addr;
4592
4593 *bad_wr = NULL;
4594 ha = dev->ha;
4595
4596 QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n",
4597 ibqp, wr, bad_wr);
4598
4599 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
4600 return -EINVAL;
4601
4602 if (qp->qp_type == IB_QPT_GSI)
4603 return qlnxr_gsi_post_send(ibqp, wr, bad_wr);
4604
4605 spin_lock_irqsave(&qp->q_lock, flags);
4606
4607 if (QLNX_IS_ROCE(dev) && (qp->state != ECORE_ROCE_QP_STATE_RTS) &&
4608 (qp->state != ECORE_ROCE_QP_STATE_ERR) &&
4609 (qp->state != ECORE_ROCE_QP_STATE_SQD)) {
4610 spin_unlock_irqrestore(&qp->q_lock, flags);
4611 *bad_wr = wr;
4612 QL_DPRINT11(ha, "QP in wrong state! QP icid=0x%x state %d\n",
4613 qp->icid, qp->state);
4614 return -EINVAL;
4615 }
4616
4617 if (!wr) {
4618 QL_DPRINT11(ha, "Got an empty post send???\n");
4619 }
4620
4621 while (wr) {
4622 struct rdma_sq_common_wqe *wqe;
4623 struct rdma_sq_send_wqe *swqe;
4624 struct rdma_sq_send_wqe *swqe2;
4625 struct rdma_sq_rdma_wqe_1st *rwqe;
4626 struct rdma_sq_rdma_wqe_2nd *rwqe2;
4627 struct rdma_sq_local_inv_wqe *iwqe;
4628 struct rdma_sq_atomic_wqe *awqe1;
4629 struct rdma_sq_atomic_wqe *awqe2;
4630 struct rdma_sq_atomic_wqe *awqe3;
4631 struct rdma_sq_fmr_wqe_1st *fwqe1;
4632
4633 if (!qlnxr_can_post_send(qp, wr)) {
4634 status = -ENOMEM;
4635 *bad_wr = wr;
4636 break;
4637 }
4638
4639 wqe = ecore_chain_produce(&qp->sq.pbl);
4640
4641 qp->wqe_wr_id[qp->sq.prod].signaled =
4642 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
4643
4644 /* common fields */
4645 wqe->flags = 0;
4646 wqe->flags |= (RDMA_SQ_SEND_WQE_COMP_FLG_MASK <<
4647 RDMA_SQ_SEND_WQE_COMP_FLG_SHIFT);
4648
4649 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, \
4650 !!(wr->send_flags & IB_SEND_SOLICITED));
4651
4652 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) ||
4653 (qp->signaled);
4654
4655 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
4656 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, \
4657 !!(wr->send_flags & IB_SEND_FENCE));
4658
4659 wqe->prev_wqe_size = qp->prev_wqe_size;
4660
4661 qp->wqe_wr_id[qp->sq.prod].opcode = qlnxr_ib_to_wc_opcode(wr->opcode);
4662
4663 switch (wr->opcode) {
4664 case IB_WR_SEND_WITH_IMM:
4665
4666 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
4667 swqe = (struct rdma_sq_send_wqe *)wqe;
4668 swqe->wqe_size = 2;
4669 swqe2 = (struct rdma_sq_send_wqe *)
4670 ecore_chain_produce(&qp->sq.pbl);
4671 swqe->inv_key_or_imm_data =
4672 cpu_to_le32(wr->ex.imm_data);
4673 swqe->length = cpu_to_le32(
4674 qlnxr_prepare_sq_send_data(dev,
4675 qp, swqe, swqe2, wr,
4676 bad_wr));
4677
4678 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
4679 qp->prev_wqe_size = swqe->wqe_size;
4680 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
4681
4682 QL_DPRINT12(ha, "SEND w/ IMM length = %d imm data=%x\n",
4683 swqe->length, wr->ex.imm_data);
4684
4685 break;
4686
4687 case IB_WR_SEND:
4688
4689 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
4690 swqe = (struct rdma_sq_send_wqe *)wqe;
4691
4692 swqe->wqe_size = 2;
4693 swqe2 = (struct rdma_sq_send_wqe *)
4694 ecore_chain_produce(&qp->sq.pbl);
4695 swqe->length = cpu_to_le32(
4696 qlnxr_prepare_sq_send_data(dev,
4697 qp, swqe, swqe2, wr,
4698 bad_wr));
4699 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
4700 qp->prev_wqe_size = swqe->wqe_size;
4701 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
4702
4703 QL_DPRINT12(ha, "SEND w/o IMM length = %d\n",
4704 swqe->length);
4705
4706 break;
4707
4708 case IB_WR_SEND_WITH_INV:
4709
4710 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
4711 swqe = (struct rdma_sq_send_wqe *)wqe;
4712 swqe2 = (struct rdma_sq_send_wqe *)
4713 ecore_chain_produce(&qp->sq.pbl);
4714 swqe->wqe_size = 2;
4715 swqe->inv_key_or_imm_data =
4716 cpu_to_le32(wr->ex.invalidate_rkey);
4717 swqe->length = cpu_to_le32(qlnxr_prepare_sq_send_data(dev,
4718 qp, swqe, swqe2, wr, bad_wr));
4719 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
4720 qp->prev_wqe_size = swqe->wqe_size;
4721 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
4722
4723 QL_DPRINT12(ha, "SEND w INVALIDATE length = %d\n",
4724 swqe->length);
4725 break;
4726
4727 case IB_WR_RDMA_WRITE_WITH_IMM:
4728
4729 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
4730 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
4731
4732 rwqe->wqe_size = 2;
4733 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
4734 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *)
4735 ecore_chain_produce(&qp->sq.pbl);
4736 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev,
4737 qp, rwqe, rwqe2, wr, bad_wr));
4738 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
4739 qp->prev_wqe_size = rwqe->wqe_size;
4740 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
4741
4742 QL_DPRINT12(ha,
4743 "RDMA WRITE w/ IMM length = %d imm data=%x\n",
4744 rwqe->length, rwqe->imm_data);
4745
4746 break;
4747
4748 case IB_WR_RDMA_WRITE:
4749
4750 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
4751 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
4752
4753 rwqe->wqe_size = 2;
4754 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *)
4755 ecore_chain_produce(&qp->sq.pbl);
4756 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev,
4757 qp, rwqe, rwqe2, wr, bad_wr));
4758 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
4759 qp->prev_wqe_size = rwqe->wqe_size;
4760 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
4761
4762 QL_DPRINT12(ha,
4763 "RDMA WRITE w/o IMM length = %d\n",
4764 rwqe->length);
4765
4766 break;
4767
4768 case IB_WR_RDMA_READ_WITH_INV:
4769
4770 QL_DPRINT12(ha,
4771 "RDMA READ WITH INVALIDATE not supported\n");
4772
4773 *bad_wr = wr;
4774 rc = -EINVAL;
4775
4776 break;
4777
4778 case IB_WR_RDMA_READ:
4779
4780 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
4781 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
4782
4783 rwqe->wqe_size = 2;
4784 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *)
4785 ecore_chain_produce(&qp->sq.pbl);
4786 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev,
4787 qp, rwqe, rwqe2, wr, bad_wr));
4788
4789 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
4790 qp->prev_wqe_size = rwqe->wqe_size;
4791 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
4792
4793 QL_DPRINT12(ha, "RDMA READ length = %d\n",
4794 rwqe->length);
4795
4796 break;
4797
4798 case IB_WR_ATOMIC_CMP_AND_SWP:
4799 case IB_WR_ATOMIC_FETCH_AND_ADD:
4800
4801 QL_DPRINT12(ha,
4802 "ATOMIC operation = %s\n",
4803 ((wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) ?
4804 "IB_WR_ATOMIC_CMP_AND_SWP" :
4805 "IB_WR_ATOMIC_FETCH_AND_ADD"));
4806
4807 awqe1 = (struct rdma_sq_atomic_wqe *)wqe;
4808 awqe1->prev_wqe_size = 4;
4809
4810 awqe2 = (struct rdma_sq_atomic_wqe *)
4811 ecore_chain_produce(&qp->sq.pbl);
4812
4813 TYPEPTR_ADDR_SET(awqe2, remote_va, \
4814 atomic_wr(wr)->remote_addr);
4815
4816 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
4817
4818 awqe3 = (struct rdma_sq_atomic_wqe *)
4819 ecore_chain_produce(&qp->sq.pbl);
4820
4821 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
4822 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
4823 TYPEPTR_ADDR_SET(awqe3, swap_data,
4824 atomic_wr(wr)->compare_add);
4825 } else {
4826 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
4827 TYPEPTR_ADDR_SET(awqe3, swap_data,
4828 atomic_wr(wr)->swap);
4829 TYPEPTR_ADDR_SET(awqe3, cmp_data,
4830 atomic_wr(wr)->compare_add);
4831 }
4832
4833 qlnxr_prepare_sq_sges(dev, qp, NULL, wr);
4834
4835 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->prev_wqe_size;
4836 qp->prev_wqe_size = awqe1->prev_wqe_size;
4837
4838 break;
4839
4840 case IB_WR_LOCAL_INV:
4841
4842 QL_DPRINT12(ha,
4843 "INVALIDATE length (IB_WR_LOCAL_INV)\n");
4844
4845 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
4846 iwqe->prev_wqe_size = 1;
4847
4848 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
4849 iwqe->inv_l_key = wr->ex.invalidate_rkey;
4850 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->prev_wqe_size;
4851 qp->prev_wqe_size = iwqe->prev_wqe_size;
4852
4853 break;
4854
4855 #if __FreeBSD_version >= 1102000
4856
4857 case IB_WR_REG_MR:
4858
4859 QL_DPRINT12(ha, "IB_WR_REG_MR\n");
4860
4861 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
4862 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
4863 fwqe1->wqe_size = 2;
4864
4865 rc = qlnxr_prepare_reg(qp, fwqe1, reg_wr(wr));
4866 if (rc) {
4867 QL_DPRINT11(ha, "IB_WR_REG_MR failed rc=%d\n", rc);
4868 *bad_wr = wr;
4869 break;
4870 }
4871
4872 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
4873 qp->prev_wqe_size = fwqe1->wqe_size;
4874
4875 break;
4876 #else
4877 case IB_WR_FAST_REG_MR:
4878
4879 QL_DPRINT12(ha, "FAST_MR (IB_WR_FAST_REG_MR)\n");
4880
4881 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
4882 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
4883 fwqe1->prev_wqe_size = 3;
4884
4885 rc = qlnxr_prepare_fmr(qp, fwqe1, wr);
4886
4887 if (rc) {
4888 QL_DPRINT12(ha,
4889 "FAST_MR (IB_WR_FAST_REG_MR) failed"
4890 " rc = %d\n", rc);
4891 *bad_wr = wr;
4892 break;
4893 }
4894
4895 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->prev_wqe_size;
4896 qp->prev_wqe_size = fwqe1->prev_wqe_size;
4897
4898 break;
4899 #endif /* #if __FreeBSD_version >= 1102000 */
4900
4901 default:
4902
4903 QL_DPRINT12(ha, "Invalid Opcode 0x%x!\n", wr->opcode);
4904
4905 rc = -EINVAL;
4906 *bad_wr = wr;
4907 break;
4908 }
4909
4910 if (*bad_wr) {
4911 /*
4912 * restore prod to its position before this WR was processed
4913 */
4914 ecore_chain_set_prod(&qp->sq.pbl,
4915 le16_to_cpu(qp->sq.db_data.data.value),
4916 wqe);
4917 /* restore prev_wqe_size */
4918 qp->prev_wqe_size = wqe->prev_wqe_size;
4919 status = rc;
4920
4921 QL_DPRINT12(ha, "failed *bad_wr = %p\n", *bad_wr);
4922 break; /* out of the loop */
4923 }
4924
4925 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
4926
4927 qlnxr_inc_sw_prod(&qp->sq);
4928
4929 qp->sq.db_data.data.value++;
4930
4931 wr = wr->next;
4932 }
4933
4934 /* Trigger doorbell
4935 * If there was a failure in the first WR then it will be triggered in
4936 * vane. However this is not harmful (as long as the producer value is
4937 * unchanged). For performance reasons we avoid checking for this
4938 * redundant doorbell.
4939 */
4940 wmb();
4941 //writel(qp->sq.db_data.raw, qp->sq.db);
4942
4943 reg_addr = (uint32_t)((uint8_t *)qp->sq.db - (uint8_t *)ha->cdev.doorbells);
4944 bus_write_4(ha->pci_dbells, reg_addr, qp->sq.db_data.raw);
4945 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ);
4946
4947 mmiowb();
4948
4949 spin_unlock_irqrestore(&qp->q_lock, flags);
4950
4951 QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n",
4952 ibqp, wr, bad_wr);
4953
4954 return status;
4955 }
4956
4957 static u32
4958 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq)
4959 {
4960 u32 used;
4961
4962 /* Calculate number of elements used based on producer
4963 * count and consumer count and subtract it from max
4964 * work request supported so that we get elements left.
4965 */
4966 used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt;
4967
4968 return hw_srq->max_wr - used;
4969 }
4970
4971 int
4972 qlnxr_post_recv(struct ib_qp *ibqp,
4973 const struct ib_recv_wr *wr,
4974 const struct ib_recv_wr **bad_wr)
4975 {
4976 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
4977 struct qlnxr_dev *dev = qp->dev;
4978 unsigned long flags;
4979 int status = 0;
4980 qlnx_host_t *ha;
4981 uint32_t reg_addr;
4982
4983 ha = dev->ha;
4984
4985 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
4986 return -EINVAL;
4987
4988 QL_DPRINT12(ha, "enter\n");
4989
4990 if (qp->qp_type == IB_QPT_GSI) {
4991 QL_DPRINT12(ha, "(qp->qp_type = IB_QPT_GSI)\n");
4992 return qlnxr_gsi_post_recv(ibqp, wr, bad_wr);
4993 }
4994
4995 if (qp->srq) {
4996 QL_DPRINT11(ha, "qp->srq [%p]"
4997 " QP is associated with SRQ, cannot post RQ buffers\n",
4998 qp->srq);
4999 return -EINVAL;
5000 }
5001
5002 spin_lock_irqsave(&qp->q_lock, flags);
5003
5004 if (qp->state == ECORE_ROCE_QP_STATE_RESET) {
5005 spin_unlock_irqrestore(&qp->q_lock, flags);
5006 *bad_wr = wr;
5007
5008 QL_DPRINT11(ha, "qp->qp_type = ECORE_ROCE_QP_STATE_RESET\n");
5009
5010 return -EINVAL;
5011 }
5012
5013 while (wr) {
5014 int i;
5015
5016 if ((ecore_chain_get_elem_left_u32(&qp->rq.pbl) <
5017 QLNXR_MAX_RQE_ELEMENTS_PER_RQE) ||
5018 (wr->num_sge > qp->rq.max_sges)) {
5019 status = -ENOMEM;
5020 *bad_wr = wr;
5021 break;
5022 }
5023 for (i = 0; i < wr->num_sge; i++) {
5024 u32 flags = 0;
5025 struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl);
5026
5027 /* first one must include the number of SGE in the list */
5028 if (!i)
5029 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, wr->num_sge);
5030
5031 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, wr->sg_list[i].lkey);
5032
5033 RQ_SGE_SET(rqe, wr->sg_list[i].addr, \
5034 wr->sg_list[i].length, flags);
5035 }
5036 /* Special case of no sges. FW requires between 1-4 sges...
5037 * in this case we need to post 1 sge with length zero. this is
5038 * because rdma write with immediate consumes an RQ. */
5039 if (!wr->num_sge) {
5040 u32 flags = 0;
5041 struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl);
5042
5043 /* first one must include the number of SGE in the list */
5044 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0);
5045 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
5046
5047 //RQ_SGE_SET(rqe, 0, 0, flags);
5048 rqe->addr.hi = 0;
5049 rqe->addr.lo = 0;
5050
5051 rqe->length = 0;
5052 rqe->flags = cpu_to_le32(flags);
5053
5054 i = 1;
5055 }
5056
5057 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
5058 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
5059
5060 qlnxr_inc_sw_prod(&qp->rq);
5061
5062 wmb();
5063
5064 qp->rq.db_data.data.value++;
5065
5066 // writel(qp->rq.db_data.raw, qp->rq.db);
5067 mmiowb();
5068 // if (QLNX_IS_IWARP(dev)) {
5069 // writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
5070 // mmiowb(); /* for second doorbell */
5071 // }
5072
5073 reg_addr = (uint32_t)((uint8_t *)qp->rq.db -
5074 (uint8_t *)ha->cdev.doorbells);
5075
5076 bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw);
5077 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ);
5078
5079 if (QLNX_IS_IWARP(dev)) {
5080 reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 -
5081 (uint8_t *)ha->cdev.doorbells);
5082 bus_write_4(ha->pci_dbells, reg_addr, \
5083 qp->rq.iwarp_db2_data.raw);
5084 bus_barrier(ha->pci_dbells, 0, 0, \
5085 BUS_SPACE_BARRIER_READ);
5086 }
5087
5088 wr = wr->next;
5089 }
5090
5091 spin_unlock_irqrestore(&qp->q_lock, flags);
5092
5093 QL_DPRINT12(ha, "exit status = 0x%x\n", status);
5094
5095 return status;
5096 }
5097
5098 /* In fmr we need to increase the number of fmr completed counter for the fmr
5099 * algorithm determining whether we can free a pbl or not.
5100 * we need to perform this whether the work request was signaled or not. for
5101 * this purpose we call this function from the condition that checks if a wr
5102 * should be skipped, to make sure we don't miss it ( possibly this fmr
5103 * operation was not signalted)
5104 */
5105 static inline void
5106 qlnxr_chk_if_fmr(struct qlnxr_qp *qp)
5107 {
5108 #if __FreeBSD_version >= 1102000
5109
5110 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
5111 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
5112 #else
5113 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_FAST_REG_MR)
5114 qp->wqe_wr_id[qp->sq.cons].frmr->info.completed++;
5115
5116 #endif /* #if __FreeBSD_version >= 1102000 */
5117 }
5118
5119 static int
5120 process_req(struct qlnxr_dev *dev,
5121 struct qlnxr_qp *qp,
5122 struct qlnxr_cq *cq,
5123 int num_entries,
5124 struct ib_wc *wc,
5125 u16 hw_cons,
5126 enum ib_wc_status status,
5127 int force)
5128 {
5129 u16 cnt = 0;
5130 qlnx_host_t *ha = dev->ha;
5131
5132 QL_DPRINT12(ha, "enter\n");
5133
5134 while (num_entries && qp->sq.wqe_cons != hw_cons) {
5135 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
5136 qlnxr_chk_if_fmr(qp);
5137 /* skip WC */
5138 goto next_cqe;
5139 }
5140
5141 /* fill WC */
5142 wc->status = status;
5143 wc->vendor_err = 0;
5144 wc->wc_flags = 0;
5145 wc->src_qp = qp->id;
5146 wc->qp = &qp->ibqp;
5147
5148 // common section
5149 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
5150 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
5151
5152 switch (wc->opcode) {
5153 case IB_WC_RDMA_WRITE:
5154
5155 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
5156
5157 QL_DPRINT12(ha,
5158 "opcode = IB_WC_RDMA_WRITE bytes = %d\n",
5159 qp->wqe_wr_id[qp->sq.cons].bytes_len);
5160 break;
5161
5162 case IB_WC_COMP_SWAP:
5163 case IB_WC_FETCH_ADD:
5164 wc->byte_len = 8;
5165 break;
5166
5167 #if __FreeBSD_version >= 1102000
5168 case IB_WC_REG_MR:
5169 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
5170 break;
5171 #else
5172 case IB_WC_FAST_REG_MR:
5173 qp->wqe_wr_id[qp->sq.cons].frmr->info.completed++;
5174 break;
5175 #endif /* #if __FreeBSD_version >= 1102000 */
5176
5177 case IB_WC_RDMA_READ:
5178 case IB_WC_SEND:
5179
5180 QL_DPRINT12(ha, "opcode = 0x%x \n", wc->opcode);
5181 break;
5182 default:
5183 ;//DP_ERR("TBD ERROR");
5184 }
5185
5186 num_entries--;
5187 wc++;
5188 cnt++;
5189 next_cqe:
5190 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
5191 ecore_chain_consume(&qp->sq.pbl);
5192 qlnxr_inc_sw_cons(&qp->sq);
5193 }
5194
5195 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
5196 return cnt;
5197 }
5198
5199 static int
5200 qlnxr_poll_cq_req(struct qlnxr_dev *dev,
5201 struct qlnxr_qp *qp,
5202 struct qlnxr_cq *cq,
5203 int num_entries,
5204 struct ib_wc *wc,
5205 struct rdma_cqe_requester *req)
5206 {
5207 int cnt = 0;
5208 qlnx_host_t *ha = dev->ha;
5209
5210 QL_DPRINT12(ha, "enter req->status = 0x%x\n", req->status);
5211
5212 switch (req->status) {
5213 case RDMA_CQE_REQ_STS_OK:
5214
5215 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
5216 IB_WC_SUCCESS, 0);
5217 break;
5218
5219 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
5220
5221 if (qp->state != ECORE_ROCE_QP_STATE_ERR)
5222 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
5223 IB_WC_WR_FLUSH_ERR, 1);
5224 break;
5225
5226 default: /* other errors case */
5227
5228 /* process all WQE before the cosumer */
5229 qp->state = ECORE_ROCE_QP_STATE_ERR;
5230 cnt = process_req(dev, qp, cq, num_entries, wc,
5231 req->sq_cons - 1, IB_WC_SUCCESS, 0);
5232 wc += cnt;
5233 /* if we have extra WC fill it with actual error info */
5234
5235 if (cnt < num_entries) {
5236 enum ib_wc_status wc_status;
5237
5238 switch (req->status) {
5239 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
5240 wc_status = IB_WC_BAD_RESP_ERR;
5241 break;
5242 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
5243 wc_status = IB_WC_LOC_LEN_ERR;
5244 break;
5245 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
5246 wc_status = IB_WC_LOC_QP_OP_ERR;
5247 break;
5248 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
5249 wc_status = IB_WC_LOC_PROT_ERR;
5250 break;
5251 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
5252 wc_status = IB_WC_MW_BIND_ERR;
5253 break;
5254 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
5255 wc_status = IB_WC_REM_INV_REQ_ERR;
5256 break;
5257 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
5258 wc_status = IB_WC_REM_ACCESS_ERR;
5259 break;
5260 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
5261 wc_status = IB_WC_REM_OP_ERR;
5262 break;
5263 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
5264 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
5265 break;
5266 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
5267 wc_status = IB_WC_RETRY_EXC_ERR;
5268 break;
5269 default:
5270 wc_status = IB_WC_GENERAL_ERR;
5271 }
5272
5273 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
5274 wc_status, 1 /* force use of WC */);
5275 }
5276 }
5277
5278 QL_DPRINT12(ha, "exit cnt = %d\n", cnt);
5279 return cnt;
5280 }
5281
5282 static void
5283 __process_resp_one(struct qlnxr_dev *dev,
5284 struct qlnxr_qp *qp,
5285 struct qlnxr_cq *cq,
5286 struct ib_wc *wc,
5287 struct rdma_cqe_responder *resp,
5288 u64 wr_id)
5289 {
5290 enum ib_wc_status wc_status = IB_WC_SUCCESS;
5291 #if __FreeBSD_version < 1102000
5292 u8 flags;
5293 #endif
5294 qlnx_host_t *ha = dev->ha;
5295
5296 QL_DPRINT12(ha, "enter qp = %p resp->status = 0x%x\n",
5297 qp, resp->status);
5298
5299 wc->opcode = IB_WC_RECV;
5300 wc->wc_flags = 0;
5301
5302 switch (resp->status) {
5303 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
5304 wc_status = IB_WC_LOC_ACCESS_ERR;
5305 break;
5306
5307 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
5308 wc_status = IB_WC_LOC_LEN_ERR;
5309 break;
5310
5311 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
5312 wc_status = IB_WC_LOC_QP_OP_ERR;
5313 break;
5314
5315 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
5316 wc_status = IB_WC_LOC_PROT_ERR;
5317 break;
5318
5319 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
5320 wc_status = IB_WC_MW_BIND_ERR;
5321 break;
5322
5323 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
5324 wc_status = IB_WC_REM_INV_RD_REQ_ERR;
5325 break;
5326
5327 case RDMA_CQE_RESP_STS_OK:
5328
5329 #if __FreeBSD_version >= 1102000
5330 if (resp->flags & QLNXR_RESP_IMM) {
5331 wc->ex.imm_data =
5332 le32_to_cpu(resp->imm_data_or_inv_r_Key);
5333 wc->wc_flags |= IB_WC_WITH_IMM;
5334
5335 if (resp->flags & QLNXR_RESP_RDMA)
5336 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
5337
5338 if (resp->flags & QLNXR_RESP_INV) {
5339 QL_DPRINT11(ha,
5340 "Invalid flags QLNXR_RESP_INV [0x%x]"
5341 "qp = %p qp->id = 0x%x cq = %p"
5342 " cq->icid = 0x%x\n",
5343 resp->flags, qp, qp->id, cq, cq->icid );
5344 }
5345 } else if (resp->flags & QLNXR_RESP_INV) {
5346 wc->ex.imm_data =
5347 le32_to_cpu(resp->imm_data_or_inv_r_Key);
5348 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
5349
5350 if (resp->flags & QLNXR_RESP_RDMA) {
5351 QL_DPRINT11(ha,
5352 "Invalid flags QLNXR_RESP_RDMA [0x%x]"
5353 "qp = %p qp->id = 0x%x cq = %p"
5354 " cq->icid = 0x%x\n",
5355 resp->flags, qp, qp->id, cq, cq->icid );
5356 }
5357 } else if (resp->flags & QLNXR_RESP_RDMA) {
5358 QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]"
5359 "qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n",
5360 resp->flags, qp, qp->id, cq, cq->icid );
5361 }
5362 #else
5363 wc_status = IB_WC_SUCCESS;
5364 wc->byte_len = le32_to_cpu(resp->length);
5365
5366 flags = resp->flags & QLNXR_RESP_RDMA_IMM;
5367
5368 switch (flags) {
5369 case QLNXR_RESP_RDMA_IMM:
5370 /* update opcode */
5371 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
5372 /* fall to set imm data */
5373 case QLNXR_RESP_IMM:
5374 wc->ex.imm_data =
5375 le32_to_cpu(resp->imm_data_or_inv_r_Key);
5376 wc->wc_flags |= IB_WC_WITH_IMM;
5377 break;
5378 case QLNXR_RESP_RDMA:
5379 QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]"
5380 "qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n",
5381 resp->flags, qp, qp->id, cq, cq->icid );
5382 break;
5383 default:
5384 /* valid configuration, but nothing todo here */
5385 ;
5386 }
5387 #endif /* #if __FreeBSD_version >= 1102000 */
5388
5389 break;
5390 default:
5391 wc_status = IB_WC_GENERAL_ERR;
5392 }
5393
5394 /* fill WC */
5395 wc->status = wc_status;
5396 wc->vendor_err = 0;
5397 wc->src_qp = qp->id;
5398 wc->qp = &qp->ibqp;
5399 wc->wr_id = wr_id;
5400
5401 QL_DPRINT12(ha, "exit status = 0x%x\n", wc_status);
5402
5403 return;
5404 }
5405
5406 static int
5407 process_resp_one_srq(struct qlnxr_dev *dev,
5408 struct qlnxr_qp *qp,
5409 struct qlnxr_cq *cq,
5410 struct ib_wc *wc,
5411 struct rdma_cqe_responder *resp)
5412 {
5413 struct qlnxr_srq *srq = qp->srq;
5414 u64 wr_id;
5415 qlnx_host_t *ha = dev->ha;
5416
5417 QL_DPRINT12(ha, "enter\n");
5418
5419 wr_id = HILO_U64(resp->srq_wr_id.hi, resp->srq_wr_id.lo);
5420
5421 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
5422 wc->status = IB_WC_WR_FLUSH_ERR;
5423 wc->vendor_err = 0;
5424 wc->wr_id = wr_id;
5425 wc->byte_len = 0;
5426 wc->src_qp = qp->id;
5427 wc->qp = &qp->ibqp;
5428 wc->wr_id = wr_id;
5429 } else {
5430 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
5431 }
5432
5433 /* PBL is maintained in case of WR granularity.
5434 * So increment WR consumer after consuming WR
5435 */
5436 srq->hw_srq.wr_cons_cnt++;
5437
5438 QL_DPRINT12(ha, "exit\n");
5439 return 1;
5440 }
5441
5442 static int
5443 process_resp_one(struct qlnxr_dev *dev,
5444 struct qlnxr_qp *qp,
5445 struct qlnxr_cq *cq,
5446 struct ib_wc *wc,
5447 struct rdma_cqe_responder *resp)
5448 {
5449 qlnx_host_t *ha = dev->ha;
5450 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
5451
5452 QL_DPRINT12(ha, "enter\n");
5453
5454 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
5455
5456 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
5457 ecore_chain_consume(&qp->rq.pbl);
5458 qlnxr_inc_sw_cons(&qp->rq);
5459
5460 QL_DPRINT12(ha, "exit\n");
5461 return 1;
5462 }
5463
5464 static int
5465 process_resp_flush(struct qlnxr_qp *qp,
5466 int num_entries,
5467 struct ib_wc *wc,
5468 u16 hw_cons)
5469 {
5470 u16 cnt = 0;
5471 qlnx_host_t *ha = qp->dev->ha;
5472
5473 QL_DPRINT12(ha, "enter\n");
5474
5475 while (num_entries && qp->rq.wqe_cons != hw_cons) {
5476 /* fill WC */
5477 wc->status = IB_WC_WR_FLUSH_ERR;
5478 wc->vendor_err = 0;
5479 wc->wc_flags = 0;
5480 wc->src_qp = qp->id;
5481 wc->byte_len = 0;
5482 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
5483 wc->qp = &qp->ibqp;
5484 num_entries--;
5485 wc++;
5486 cnt++;
5487 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
5488 ecore_chain_consume(&qp->rq.pbl);
5489 qlnxr_inc_sw_cons(&qp->rq);
5490 }
5491
5492 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
5493 return cnt;
5494 }
5495
5496 static void
5497 try_consume_resp_cqe(struct qlnxr_cq *cq,
5498 struct qlnxr_qp *qp,
5499 struct rdma_cqe_responder *resp,
5500 int *update)
5501 {
5502 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) {
5503 consume_cqe(cq);
5504 *update |= 1;
5505 }
5506 }
5507
5508 static int
5509 qlnxr_poll_cq_resp_srq(struct qlnxr_dev *dev,
5510 struct qlnxr_qp *qp,
5511 struct qlnxr_cq *cq,
5512 int num_entries,
5513 struct ib_wc *wc,
5514 struct rdma_cqe_responder *resp,
5515 int *update)
5516 {
5517 int cnt;
5518 qlnx_host_t *ha = dev->ha;
5519
5520 QL_DPRINT12(ha, "enter\n");
5521
5522 cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
5523 consume_cqe(cq);
5524 *update |= 1;
5525
5526 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
5527 return cnt;
5528 }
5529
5530 static int
5531 qlnxr_poll_cq_resp(struct qlnxr_dev *dev,
5532 struct qlnxr_qp *qp,
5533 struct qlnxr_cq *cq,
5534 int num_entries,
5535 struct ib_wc *wc,
5536 struct rdma_cqe_responder *resp,
5537 int *update)
5538 {
5539 int cnt;
5540 qlnx_host_t *ha = dev->ha;
5541
5542 QL_DPRINT12(ha, "enter\n");
5543
5544 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
5545 cnt = process_resp_flush(qp, num_entries, wc,
5546 resp->rq_cons);
5547 try_consume_resp_cqe(cq, qp, resp, update);
5548 } else {
5549 cnt = process_resp_one(dev, qp, cq, wc, resp);
5550 consume_cqe(cq);
5551 *update |= 1;
5552 }
5553
5554 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
5555 return cnt;
5556 }
5557
5558 static void
5559 try_consume_req_cqe(struct qlnxr_cq *cq, struct qlnxr_qp *qp,
5560 struct rdma_cqe_requester *req, int *update)
5561 {
5562 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
5563 consume_cqe(cq);
5564 *update |= 1;
5565 }
5566 }
5567
5568 static void
5569 doorbell_cq(struct qlnxr_dev *dev, struct qlnxr_cq *cq, u32 cons, u8 flags)
5570 {
5571 uint64_t reg_addr;
5572 qlnx_host_t *ha = dev->ha;
5573
5574 QL_DPRINT12(ha, "enter\n");
5575
5576 wmb();
5577 cq->db.data.agg_flags = flags;
5578 cq->db.data.value = cpu_to_le32(cons);
5579
5580 reg_addr = (uint64_t)((uint8_t *)cq->db_addr -
5581 (uint8_t *)(ha->cdev.doorbells));
5582
5583 bus_write_8(ha->pci_dbells, reg_addr, cq->db.raw);
5584 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ);
5585
5586 QL_DPRINT12(ha, "exit\n");
5587 return;
5588
5589 //#ifdef __LP64__
5590 // writeq(cq->db.raw, cq->db_addr);
5591 //#else
5592 /* Note that since the FW allows 64 bit write only, in 32bit systems
5593 * the value of db_addr must be low enough. This is currently not
5594 * enforced.
5595 */
5596 // writel(cq->db.raw & 0xffffffff, cq->db_addr);
5597 // mmiowb();
5598 //#endif
5599 }
5600
5601 static int
5602 is_valid_cqe(struct qlnxr_cq *cq, union rdma_cqe *cqe)
5603 {
5604 struct rdma_cqe_requester *resp_cqe = &cqe->req;
5605 return (resp_cqe->flags & RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK) ==
5606 cq->pbl_toggle;
5607 }
5608
5609 int
5610 qlnxr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
5611 {
5612 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq);
5613 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device));
5614 int done = 0;
5615 union rdma_cqe *cqe = cq->latest_cqe;
5616 int update = 0;
5617 u32 old_cons, new_cons;
5618 unsigned long flags;
5619 qlnx_host_t *ha = dev->ha;
5620
5621 QL_DPRINT12(ha, "enter\n");
5622
5623 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
5624 return -EINVAL;
5625
5626 if (cq->destroyed) {
5627 QL_DPRINT11(ha, "called after destroy for cq %p (icid=%d)\n",
5628 cq, cq->icid);
5629 return 0;
5630 }
5631
5632 if (cq->cq_type == QLNXR_CQ_TYPE_GSI)
5633 return qlnxr_gsi_poll_cq(ibcq, num_entries, wc);
5634
5635 spin_lock_irqsave(&cq->cq_lock, flags);
5636
5637 old_cons = ecore_chain_get_cons_idx_u32(&cq->pbl);
5638
5639 while (num_entries && is_valid_cqe(cq, cqe)) {
5640 int cnt = 0;
5641 struct qlnxr_qp *qp;
5642 struct rdma_cqe_requester *resp_cqe;
5643 enum rdma_cqe_type cqe_type;
5644
5645 /* prevent speculative reads of any field of CQE */
5646 rmb();
5647
5648 resp_cqe = &cqe->req;
5649 qp = (struct qlnxr_qp *)(uintptr_t)HILO_U64(resp_cqe->qp_handle.hi,
5650 resp_cqe->qp_handle.lo);
5651
5652 if (!qp) {
5653 QL_DPRINT11(ha, "qp = NULL\n");
5654 break;
5655 }
5656
5657 wc->qp = &qp->ibqp;
5658
5659 cqe_type = GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
5660
5661 switch (cqe_type) {
5662 case RDMA_CQE_TYPE_REQUESTER:
5663 cnt = qlnxr_poll_cq_req(dev, qp, cq, num_entries,
5664 wc, &cqe->req);
5665 try_consume_req_cqe(cq, qp, &cqe->req, &update);
5666 break;
5667 case RDMA_CQE_TYPE_RESPONDER_RQ:
5668 cnt = qlnxr_poll_cq_resp(dev, qp, cq, num_entries,
5669 wc, &cqe->resp, &update);
5670 break;
5671 case RDMA_CQE_TYPE_RESPONDER_SRQ:
5672 cnt = qlnxr_poll_cq_resp_srq(dev, qp, cq, num_entries,
5673 wc, &cqe->resp, &update);
5674 break;
5675 case RDMA_CQE_TYPE_INVALID:
5676 default:
5677 QL_DPRINT11(ha, "cqe type [0x%x] invalid\n", cqe_type);
5678 break;
5679 }
5680 num_entries -= cnt;
5681 wc += cnt;
5682 done += cnt;
5683
5684 cqe = cq->latest_cqe;
5685 }
5686 new_cons = ecore_chain_get_cons_idx_u32(&cq->pbl);
5687
5688 cq->cq_cons += new_cons - old_cons;
5689
5690 if (update) {
5691 /* doorbell notifies abount latest VALID entry,
5692 * but chain already point to the next INVALID one
5693 */
5694 doorbell_cq(dev, cq, cq->cq_cons - 1, cq->arm_flags);
5695 QL_DPRINT12(ha, "cq = %p cons = 0x%x "
5696 "arm_flags = 0x%x db.icid = 0x%x\n", cq,
5697 (cq->cq_cons - 1), cq->arm_flags, cq->db.data.icid);
5698 }
5699
5700 spin_unlock_irqrestore(&cq->cq_lock, flags);
5701
5702 QL_DPRINT12(ha, "exit\n");
5703
5704 return done;
5705 }
5706
5707 int
5708 qlnxr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
5709 {
5710 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq);
5711 unsigned long sflags;
5712 struct qlnxr_dev *dev;
5713 qlnx_host_t *ha;
5714
5715 dev = get_qlnxr_dev((ibcq->device));
5716 ha = dev->ha;
5717
5718 QL_DPRINT12(ha, "enter ibcq = %p flags = 0x%x "
5719 "cp = %p cons = 0x%x cq_type = 0x%x\n", ibcq,
5720 flags, cq, cq->cq_cons, cq->cq_type);
5721
5722 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
5723 return -EINVAL;
5724
5725 if (cq->destroyed) {
5726 QL_DPRINT11(ha, "cq was already destroyed cq = %p icid=%d\n",
5727 cq, cq->icid);
5728 return -EINVAL;
5729 }
5730
5731 if (cq->cq_type == QLNXR_CQ_TYPE_GSI) {
5732 return 0;
5733 }
5734
5735 spin_lock_irqsave(&cq->cq_lock, sflags);
5736
5737 cq->arm_flags = 0;
5738
5739 if (flags & IB_CQ_SOLICITED) {
5740 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
5741 }
5742 if (flags & IB_CQ_NEXT_COMP) {
5743 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
5744 }
5745
5746 doorbell_cq(dev, cq, (cq->cq_cons - 1), cq->arm_flags);
5747
5748 spin_unlock_irqrestore(&cq->cq_lock, sflags);
5749
5750 QL_DPRINT12(ha, "exit ibcq = %p flags = 0x%x\n", ibcq, flags);
5751 return 0;
5752 }
5753
5754 static struct qlnxr_mr *
5755 __qlnxr_alloc_mr(struct ib_pd *ibpd, int max_page_list_len)
5756 {
5757 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
5758 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
5759 struct qlnxr_mr *mr;
5760 int rc = -ENOMEM;
5761 qlnx_host_t *ha;
5762
5763 ha = dev->ha;
5764
5765 QL_DPRINT12(ha, "enter ibpd = %p pd = %p "
5766 " pd_id = %d max_page_list_len = %d\n",
5767 ibpd, pd, pd->pd_id, max_page_list_len);
5768
5769 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
5770 if (!mr) {
5771 QL_DPRINT11(ha, "kzalloc(mr) failed\n");
5772 return ERR_PTR(rc);
5773 }
5774
5775 mr->dev = dev;
5776 mr->type = QLNXR_MR_FRMR;
5777
5778 rc = qlnxr_init_mr_info(dev, &mr->info, max_page_list_len,
5779 1 /* allow dual layer pbl */);
5780 if (rc) {
5781 QL_DPRINT11(ha, "qlnxr_init_mr_info failed\n");
5782 goto err0;
5783 }
5784
5785 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
5786 if (rc) {
5787 QL_DPRINT11(ha, "ecore_rdma_alloc_tid failed\n");
5788 goto err0;
5789 }
5790
5791 /* index only, 18 bit long, lkey = itid << 8 | key */
5792 mr->hw_mr.tid_type = ECORE_RDMA_TID_FMR;
5793 mr->hw_mr.key = 0;
5794 mr->hw_mr.pd = pd->pd_id;
5795 mr->hw_mr.local_read = 1;
5796 mr->hw_mr.local_write = 0;
5797 mr->hw_mr.remote_read = 0;
5798 mr->hw_mr.remote_write = 0;
5799 mr->hw_mr.remote_atomic = 0;
5800 mr->hw_mr.mw_bind = false; /* TBD MW BIND */
5801 mr->hw_mr.pbl_ptr = 0; /* Will be supplied during post */
5802 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
5803 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
5804 mr->hw_mr.fbo = 0;
5805 mr->hw_mr.length = 0;
5806 mr->hw_mr.vaddr = 0;
5807 mr->hw_mr.zbva = false; /* TBD figure when this should be true */
5808 mr->hw_mr.phy_mr = true; /* Fast MR - True, Regular Register False */
5809 mr->hw_mr.dma_mr = false;
5810
5811 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
5812 if (rc) {
5813 QL_DPRINT11(ha, "ecore_rdma_register_tid failed\n");
5814 goto err1;
5815 }
5816
5817 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
5818 mr->ibmr.rkey = mr->ibmr.lkey;
5819
5820 QL_DPRINT12(ha, "exit mr = %p mr->ibmr.lkey = 0x%x\n",
5821 mr, mr->ibmr.lkey);
5822
5823 return mr;
5824
5825 err1:
5826 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
5827 err0:
5828 kfree(mr);
5829
5830 QL_DPRINT12(ha, "exit\n");
5831
5832 return ERR_PTR(rc);
5833 }
5834
5835 #if __FreeBSD_version >= 1102000
5836
5837 struct ib_mr *
5838 qlnxr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, u32 max_num_sg)
5839 {
5840 struct qlnxr_dev *dev;
5841 struct qlnxr_mr *mr;
5842 qlnx_host_t *ha;
5843
5844 dev = get_qlnxr_dev(ibpd->device);
5845 ha = dev->ha;
5846
5847 QL_DPRINT12(ha, "enter\n");
5848
5849 if (mr_type != IB_MR_TYPE_MEM_REG)
5850 return ERR_PTR(-EINVAL);
5851
5852 mr = __qlnxr_alloc_mr(ibpd, max_num_sg);
5853
5854 if (IS_ERR(mr))
5855 return ERR_PTR(-EINVAL);
5856
5857 QL_DPRINT12(ha, "exit mr = %p &mr->ibmr = %p\n", mr, &mr->ibmr);
5858
5859 return &mr->ibmr;
5860 }
5861
5862 static int
5863 qlnxr_set_page(struct ib_mr *ibmr, u64 addr)
5864 {
5865 struct qlnxr_mr *mr = get_qlnxr_mr(ibmr);
5866 struct qlnxr_pbl *pbl_table;
5867 struct regpair *pbe;
5868 struct qlnxr_dev *dev;
5869 qlnx_host_t *ha;
5870 u32 pbes_in_page;
5871
5872 dev = mr->dev;
5873 ha = dev->ha;
5874
5875 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
5876 QL_DPRINT12(ha, "fails mr->npages %d\n", mr->npages);
5877 return -ENOMEM;
5878 }
5879
5880 QL_DPRINT12(ha, "mr->npages %d addr = %p enter\n", mr->npages,
5881 ((void *)addr));
5882
5883 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
5884 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
5885 pbe = (struct regpair *)pbl_table->va;
5886 pbe += mr->npages % pbes_in_page;
5887 pbe->lo = cpu_to_le32((u32)addr);
5888 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
5889
5890 mr->npages++;
5891
5892 QL_DPRINT12(ha, "mr->npages %d addr = %p exit \n", mr->npages,
5893 ((void *)addr));
5894 return 0;
5895 }
5896
5897 int
5898 qlnxr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
5899 int sg_nents, unsigned int *sg_offset)
5900 {
5901 int ret;
5902 struct qlnxr_mr *mr = get_qlnxr_mr(ibmr);
5903 qlnx_host_t *ha;
5904
5905 if (mr == NULL)
5906 return (-1);
5907
5908 if (mr->dev == NULL)
5909 return (-1);
5910
5911 ha = mr->dev->ha;
5912
5913 QL_DPRINT12(ha, "enter\n");
5914
5915 mr->npages = 0;
5916 qlnx_handle_completed_mrs(mr->dev, &mr->info);
5917
5918 ret = ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qlnxr_set_page);
5919
5920 QL_DPRINT12(ha, "exit ret = %d\n", ret);
5921
5922 return (ret);
5923 }
5924
5925 #else
5926
5927 struct ib_mr *
5928 qlnxr_alloc_frmr(struct ib_pd *ibpd, int max_page_list_len)
5929 {
5930 struct qlnxr_dev *dev;
5931 struct qlnxr_mr *mr;
5932 qlnx_host_t *ha;
5933 struct ib_mr *ibmr = NULL;
5934
5935 dev = get_qlnxr_dev((ibpd->device));
5936 ha = dev->ha;
5937
5938 QL_DPRINT12(ha, "enter\n");
5939
5940 mr = __qlnxr_alloc_mr(ibpd, max_page_list_len);
5941
5942 if (IS_ERR(mr)) {
5943 ibmr = ERR_PTR(-EINVAL);
5944 } else {
5945 ibmr = &mr->ibmr;
5946 }
5947
5948 QL_DPRINT12(ha, "exit %p\n", ibmr);
5949 return (ibmr);
5950 }
5951
5952 void
5953 qlnxr_free_frmr_page_list(struct ib_fast_reg_page_list *page_list)
5954 {
5955 struct qlnxr_fast_reg_page_list *frmr_list;
5956
5957 frmr_list = get_qlnxr_frmr_list(page_list);
5958
5959 free_mr_info(frmr_list->dev, &frmr_list->info);
5960
5961 kfree(frmr_list->ibfrpl.page_list);
5962 kfree(frmr_list);
5963
5964 return;
5965 }
5966
5967 struct ib_fast_reg_page_list *
5968 qlnxr_alloc_frmr_page_list(struct ib_device *ibdev, int page_list_len)
5969 {
5970 struct qlnxr_fast_reg_page_list *frmr_list = NULL;
5971 struct qlnxr_dev *dev;
5972 int size = page_list_len * sizeof(u64);
5973 int rc = -ENOMEM;
5974 qlnx_host_t *ha;
5975
5976 dev = get_qlnxr_dev(ibdev);
5977 ha = dev->ha;
5978
5979 QL_DPRINT12(ha, "enter\n");
5980
5981 frmr_list = kzalloc(sizeof(*frmr_list), GFP_KERNEL);
5982 if (!frmr_list) {
5983 QL_DPRINT11(ha, "kzalloc(frmr_list) failed\n");
5984 goto err;
5985 }
5986
5987 frmr_list->dev = dev;
5988 frmr_list->ibfrpl.page_list = kzalloc(size, GFP_KERNEL);
5989 if (!frmr_list->ibfrpl.page_list) {
5990 QL_DPRINT11(ha, "frmr_list->ibfrpl.page_list = NULL failed\n");
5991 goto err0;
5992 }
5993
5994 rc = qlnxr_init_mr_info(dev, &frmr_list->info, page_list_len,
5995 1 /* allow dual layer pbl */);
5996 if (rc)
5997 goto err1;
5998
5999 QL_DPRINT12(ha, "exit %p\n", &frmr_list->ibfrpl);
6000
6001 return &frmr_list->ibfrpl;
6002
6003 err1:
6004 kfree(frmr_list->ibfrpl.page_list);
6005 err0:
6006 kfree(frmr_list);
6007 err:
6008 QL_DPRINT12(ha, "exit with error\n");
6009
6010 return ERR_PTR(rc);
6011 }
6012
6013 static int
6014 qlnxr_validate_phys_buf_list(qlnx_host_t *ha, struct ib_phys_buf *buf_list,
6015 int buf_cnt, uint64_t *total_size)
6016 {
6017 u64 size = 0;
6018
6019 *total_size = 0;
6020
6021 if (!buf_cnt || buf_list == NULL) {
6022 QL_DPRINT11(ha,
6023 "failed buf_list = %p buf_cnt = %d\n", buf_list, buf_cnt);
6024 return (-1);
6025 }
6026
6027 size = buf_list->size;
6028
6029 if (!size) {
6030 QL_DPRINT11(ha,
6031 "failed buf_list = %p buf_cnt = %d"
6032 " buf_list->size = 0\n", buf_list, buf_cnt);
6033 return (-1);
6034 }
6035
6036 while (buf_cnt) {
6037 *total_size += buf_list->size;
6038
6039 if (buf_list->size != size) {
6040 QL_DPRINT11(ha,
6041 "failed buf_list = %p buf_cnt = %d"
6042 " all buffers should have same size\n",
6043 buf_list, buf_cnt);
6044 return (-1);
6045 }
6046
6047 buf_list++;
6048 buf_cnt--;
6049 }
6050 return (0);
6051 }
6052
6053 static size_t
6054 qlnxr_get_num_pages(qlnx_host_t *ha, struct ib_phys_buf *buf_list,
6055 int buf_cnt)
6056 {
6057 int i;
6058 size_t num_pages = 0;
6059 u64 size;
6060
6061 for (i = 0; i < buf_cnt; i++) {
6062 size = 0;
6063 while (size < buf_list->size) {
6064 size += PAGE_SIZE;
6065 num_pages++;
6066 }
6067 buf_list++;
6068 }
6069 return (num_pages);
6070 }
6071
6072 static void
6073 qlnxr_populate_phys_mem_pbls(struct qlnxr_dev *dev,
6074 struct ib_phys_buf *buf_list, int buf_cnt,
6075 struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info)
6076 {
6077 struct regpair *pbe;
6078 struct qlnxr_pbl *pbl_tbl;
6079 int pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
6080 qlnx_host_t *ha;
6081 int i;
6082 u64 pbe_addr;
6083
6084 ha = dev->ha;
6085
6086 QL_DPRINT12(ha, "enter\n");
6087
6088 if (!pbl_info) {
6089 QL_DPRINT11(ha, "PBL_INFO not initialized\n");
6090 return;
6091 }
6092
6093 if (!pbl_info->num_pbes) {
6094 QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n");
6095 return;
6096 }
6097
6098 /* If we have a two layered pbl, the first pbl points to the rest
6099 * of the pbls and the first entry lays on the second pbl in the table
6100 */
6101 if (pbl_info->two_layered)
6102 pbl_tbl = &pbl[1];
6103 else
6104 pbl_tbl = pbl;
6105
6106 pbe = (struct regpair *)pbl_tbl->va;
6107 if (!pbe) {
6108 QL_DPRINT12(ha, "pbe is NULL\n");
6109 return;
6110 }
6111
6112 pbe_cnt = 0;
6113
6114 for (i = 0; i < buf_cnt; i++) {
6115 pages = buf_list->size >> PAGE_SHIFT;
6116
6117 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
6118 /* store the page address in pbe */
6119
6120 pbe_addr = buf_list->addr + (PAGE_SIZE * pg_cnt);
6121
6122 pbe->lo = cpu_to_le32((u32)pbe_addr);
6123 pbe->hi = cpu_to_le32(((u32)(pbe_addr >> 32)));
6124
6125 QL_DPRINT12(ha, "Populate pbl table:"
6126 " pbe->addr=0x%x:0x%x "
6127 " pbe_cnt = %d total_num_pbes=%d"
6128 " pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt,
6129 total_num_pbes, pbe);
6130
6131 pbe_cnt ++;
6132 total_num_pbes ++;
6133 pbe++;
6134
6135 if (total_num_pbes == pbl_info->num_pbes)
6136 return;
6137
6138 /* if the given pbl is full storing the pbes,
6139 * move to next pbl. */
6140
6141 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
6142 pbl_tbl++;
6143 pbe = (struct regpair *)pbl_tbl->va;
6144 pbe_cnt = 0;
6145 }
6146 }
6147 buf_list++;
6148 }
6149 QL_DPRINT12(ha, "exit\n");
6150 return;
6151 }
6152
6153 struct ib_mr *
6154 qlnxr_reg_kernel_mr(struct ib_pd *ibpd,
6155 struct ib_phys_buf *buf_list,
6156 int buf_cnt, int acc, u64 *iova_start)
6157 {
6158 int rc = -ENOMEM;
6159 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
6160 struct qlnxr_mr *mr;
6161 struct qlnxr_pd *pd;
6162 qlnx_host_t *ha;
6163 size_t num_pages = 0;
6164 uint64_t length;
6165
6166 ha = dev->ha;
6167
6168 QL_DPRINT12(ha, "enter\n");
6169
6170 pd = get_qlnxr_pd(ibpd);
6171
6172 QL_DPRINT12(ha, "pd = %d buf_list = %p, buf_cnt = %d,"
6173 " iova_start = %p, acc = %d\n",
6174 pd->pd_id, buf_list, buf_cnt, iova_start, acc);
6175
6176 //if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {
6177 // QL_DPRINT11(ha, "(acc & IB_ACCESS_REMOTE_WRITE &&"
6178 // " !(acc & IB_ACCESS_LOCAL_WRITE))\n");
6179 // return ERR_PTR(-EINVAL);
6180 //}
6181
6182 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
6183 if (!mr) {
6184 QL_DPRINT11(ha, "kzalloc(mr) failed\n");
6185 return ERR_PTR(rc);
6186 }
6187
6188 mr->type = QLNXR_MR_KERNEL;
6189 mr->iova_start = iova_start;
6190
6191 rc = qlnxr_validate_phys_buf_list(ha, buf_list, buf_cnt, &length);
6192 if (rc)
6193 goto err0;
6194
6195 num_pages = qlnxr_get_num_pages(ha, buf_list, buf_cnt);
6196 if (!num_pages)
6197 goto err0;
6198
6199 rc = qlnxr_init_mr_info(dev, &mr->info, num_pages, 1);
6200 if (rc) {
6201 QL_DPRINT11(ha,
6202 "qlnxr_init_mr_info failed [%d]\n", rc);
6203 goto err1;
6204 }
6205
6206 qlnxr_populate_phys_mem_pbls(dev, buf_list, buf_cnt, mr->info.pbl_table,
6207 &mr->info.pbl_info);
6208
6209 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
6210
6211 if (rc) {
6212 QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc);
6213 goto err1;
6214 }
6215
6216 /* index only, 18 bit long, lkey = itid << 8 | key */
6217 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR;
6218 mr->hw_mr.key = 0;
6219 mr->hw_mr.pd = pd->pd_id;
6220 mr->hw_mr.local_read = 1;
6221 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
6222 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
6223 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
6224 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
6225 mr->hw_mr.mw_bind = false; /* TBD MW BIND */
6226 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
6227 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
6228 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
6229 mr->hw_mr.page_size_log = ilog2(PAGE_SIZE); /* for the MR pages */
6230
6231 mr->hw_mr.fbo = 0;
6232
6233 mr->hw_mr.length = length;
6234 mr->hw_mr.vaddr = (uint64_t)iova_start;
6235 mr->hw_mr.zbva = false; /* TBD figure when this should be true */
6236 mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */
6237 mr->hw_mr.dma_mr = false;
6238
6239 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
6240 if (rc) {
6241 QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc);
6242 goto err2;
6243 }
6244
6245 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
6246 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
6247 mr->hw_mr.remote_atomic)
6248 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
6249
6250 QL_DPRINT12(ha, "lkey: %x\n", mr->ibmr.lkey);
6251
6252 return (&mr->ibmr);
6253
6254 err2:
6255 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
6256 err1:
6257 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
6258 err0:
6259 kfree(mr);
6260
6261 QL_DPRINT12(ha, "exit [%d]\n", rc);
6262 return (ERR_PTR(rc));
6263 }
6264
6265 #endif /* #if __FreeBSD_version >= 1102000 */
6266
6267 struct ib_ah *
6268 #if __FreeBSD_version >= 1102000
6269 qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr,
6270 struct ib_udata *udata)
6271 #else
6272 qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr)
6273 #endif /* #if __FreeBSD_version >= 1102000 */
6274 {
6275 struct qlnxr_dev *dev;
6276 qlnx_host_t *ha;
6277 struct qlnxr_ah *ah;
6278
6279 dev = get_qlnxr_dev((ibpd->device));
6280 ha = dev->ha;
6281
6282 QL_DPRINT12(ha, "in create_ah\n");
6283
6284 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
6285 if (!ah) {
6286 QL_DPRINT12(ha, "no address handle can be allocated\n");
6287 return ERR_PTR(-ENOMEM);
6288 }
6289
6290 ah->attr = *attr;
6291
6292 return &ah->ibah;
6293 }
6294
6295 int
6296 qlnxr_destroy_ah(struct ib_ah *ibah)
6297 {
6298 struct qlnxr_dev *dev;
6299 qlnx_host_t *ha;
6300 struct qlnxr_ah *ah = get_qlnxr_ah(ibah);
6301
6302 dev = get_qlnxr_dev((ibah->device));
6303 ha = dev->ha;
6304
6305 QL_DPRINT12(ha, "in destroy_ah\n");
6306
6307 kfree(ah);
6308 return 0;
6309 }
6310
6311 int
6312 qlnxr_query_ah(struct ib_ah *ibah, struct ib_ah_attr *attr)
6313 {
6314 struct qlnxr_dev *dev;
6315 qlnx_host_t *ha;
6316
6317 dev = get_qlnxr_dev((ibah->device));
6318 ha = dev->ha;
6319 QL_DPRINT12(ha, "Query AH not supported\n");
6320 return -EINVAL;
6321 }
6322
6323 int
6324 qlnxr_modify_ah(struct ib_ah *ibah, struct ib_ah_attr *attr)
6325 {
6326 struct qlnxr_dev *dev;
6327 qlnx_host_t *ha;
6328
6329 dev = get_qlnxr_dev((ibah->device));
6330 ha = dev->ha;
6331 QL_DPRINT12(ha, "Modify AH not supported\n");
6332 return -ENOSYS;
6333 }
6334
6335 #if __FreeBSD_version >= 1102000
6336 int
6337 qlnxr_process_mad(struct ib_device *ibdev,
6338 int process_mad_flags,
6339 u8 port_num,
6340 const struct ib_wc *in_wc,
6341 const struct ib_grh *in_grh,
6342 const struct ib_mad_hdr *mad_hdr,
6343 size_t in_mad_size,
6344 struct ib_mad_hdr *out_mad,
6345 size_t *out_mad_size,
6346 u16 *out_mad_pkey_index)
6347
6348 #else
6349
6350 int
6351 qlnxr_process_mad(struct ib_device *ibdev,
6352 int process_mad_flags,
6353 u8 port_num,
6354 struct ib_wc *in_wc,
6355 struct ib_grh *in_grh,
6356 struct ib_mad *in_mad,
6357 struct ib_mad *out_mad)
6358
6359 #endif /* #if __FreeBSD_version >= 1102000 */
6360 {
6361 struct qlnxr_dev *dev;
6362 qlnx_host_t *ha;
6363
6364 dev = get_qlnxr_dev(ibdev);
6365 ha = dev->ha;
6366 QL_DPRINT12(ha, "process mad not supported\n");
6367
6368 return -ENOSYS;
6369 // QL_DPRINT12(ha, "qlnxr_process_mad in_mad %x %x %x %x %x %x %x %x\n",
6370 // in_mad->mad_hdr.attr_id, in_mad->mad_hdr.base_version,
6371 // in_mad->mad_hdr.attr_mod, in_mad->mad_hdr.class_specific,
6372 // in_mad->mad_hdr.class_version, in_mad->mad_hdr.method,
6373 // in_mad->mad_hdr.mgmt_class, in_mad->mad_hdr.status);
6374
6375 // return IB_MAD_RESULT_SUCCESS;
6376 }
6377
6378 #if __FreeBSD_version >= 1102000
6379 int
6380 qlnxr_get_port_immutable(struct ib_device *ibdev, u8 port_num,
6381 struct ib_port_immutable *immutable)
6382 {
6383 struct qlnxr_dev *dev;
6384 qlnx_host_t *ha;
6385 struct ib_port_attr attr;
6386 int err;
6387
6388 dev = get_qlnxr_dev(ibdev);
6389 ha = dev->ha;
6390
6391 QL_DPRINT12(ha, "enter\n");
6392
6393 err = qlnxr_query_port(ibdev, port_num, &attr);
6394 if (err)
6395 return err;
6396
6397 if (QLNX_IS_IWARP(dev)) {
6398 immutable->pkey_tbl_len = 1;
6399 immutable->gid_tbl_len = 1;
6400 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
6401 immutable->max_mad_size = 0;
6402 } else {
6403 immutable->pkey_tbl_len = attr.pkey_tbl_len;
6404 immutable->gid_tbl_len = attr.gid_tbl_len;
6405 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
6406 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
6407 }
6408
6409 QL_DPRINT12(ha, "exit\n");
6410 return 0;
6411 }
6412 #endif /* #if __FreeBSD_version > 1102000 */
6413
6414 /***** iWARP related functions *************/
6415
6416 static void
6417 qlnxr_iw_mpa_request(void *context,
6418 struct ecore_iwarp_cm_event_params *params)
6419 {
6420 struct qlnxr_iw_listener *listener = (struct qlnxr_iw_listener *)context;
6421 struct qlnxr_dev *dev = listener->dev;
6422 struct qlnxr_iw_ep *ep;
6423 struct iw_cm_event event;
6424 struct sockaddr_in *laddr;
6425 struct sockaddr_in *raddr;
6426 qlnx_host_t *ha;
6427
6428 ha = dev->ha;
6429
6430 QL_DPRINT12(ha, "enter\n");
6431
6432 if (params->cm_info->ip_version != ECORE_TCP_IPV4) {
6433 QL_DPRINT11(ha, "only IPv4 supported [0x%x]\n",
6434 params->cm_info->ip_version);
6435 return;
6436 }
6437
6438 ep = kzalloc(sizeof(*ep), GFP_ATOMIC);
6439
6440 if (!ep) {
6441 QL_DPRINT11(ha, "kzalloc{ep) failed\n");
6442 return;
6443 }
6444
6445 ep->dev = dev;
6446 ep->ecore_context = params->ep_context;
6447
6448 memset(&event, 0, sizeof(event));
6449
6450 event.event = IW_CM_EVENT_CONNECT_REQUEST;
6451 event.status = params->status;
6452
6453 laddr = (struct sockaddr_in *)&event.local_addr;
6454 raddr = (struct sockaddr_in *)&event.remote_addr;
6455
6456 laddr->sin_family = AF_INET;
6457 raddr->sin_family = AF_INET;
6458
6459 laddr->sin_port = htons(params->cm_info->local_port);
6460 raddr->sin_port = htons(params->cm_info->remote_port);
6461
6462 laddr->sin_addr.s_addr = htonl(params->cm_info->local_ip[0]);
6463 raddr->sin_addr.s_addr = htonl(params->cm_info->remote_ip[0]);
6464
6465 event.provider_data = (void *)ep;
6466 event.private_data = (void *)params->cm_info->private_data;
6467 event.private_data_len = (u8)params->cm_info->private_data_len;
6468
6469 #if __FreeBSD_version >= 1100000
6470 event.ord = params->cm_info->ord;
6471 event.ird = params->cm_info->ird;
6472 #endif /* #if __FreeBSD_version >= 1100000 */
6473
6474 listener->cm_id->event_handler(listener->cm_id, &event);
6475
6476 QL_DPRINT12(ha, "exit\n");
6477
6478 return;
6479 }
6480
6481 static void
6482 qlnxr_iw_issue_event(void *context,
6483 struct ecore_iwarp_cm_event_params *params,
6484 enum iw_cm_event_type event_type,
6485 char *str)
6486 {
6487 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6488 struct qlnxr_dev *dev = ep->dev;
6489 struct iw_cm_event event;
6490 qlnx_host_t *ha;
6491
6492 ha = dev->ha;
6493
6494 QL_DPRINT12(ha, "enter\n");
6495
6496 memset(&event, 0, sizeof(event));
6497 event.status = params->status;
6498 event.event = event_type;
6499
6500 if (params->cm_info != NULL) {
6501 #if __FreeBSD_version >= 1100000
6502 event.ird = params->cm_info->ird;
6503 event.ord = params->cm_info->ord;
6504 QL_DPRINT12(ha, "ord=[%d] \n", event.ord);
6505 QL_DPRINT12(ha, "ird=[%d] \n", event.ird);
6506 #endif /* #if __FreeBSD_version >= 1100000 */
6507
6508 event.private_data_len = params->cm_info->private_data_len;
6509 event.private_data = (void *)params->cm_info->private_data;
6510 QL_DPRINT12(ha, "private_data_len=[%d] \n",
6511 event.private_data_len);
6512 }
6513
6514 QL_DPRINT12(ha, "event=[%d] %s\n", event.event, str);
6515 QL_DPRINT12(ha, "status=[%d] \n", event.status);
6516
6517 if (ep) {
6518 if (ep->cm_id)
6519 ep->cm_id->event_handler(ep->cm_id, &event);
6520 else
6521 QL_DPRINT11(ha, "ep->cm_id == NULL \n");
6522 } else {
6523 QL_DPRINT11(ha, "ep == NULL \n");
6524 }
6525
6526 QL_DPRINT12(ha, "exit\n");
6527
6528 return;
6529 }
6530
6531 static void
6532 qlnxr_iw_close_event(void *context,
6533 struct ecore_iwarp_cm_event_params *params)
6534 {
6535 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6536 struct qlnxr_dev *dev = ep->dev;
6537 qlnx_host_t *ha;
6538
6539 ha = dev->ha;
6540
6541 QL_DPRINT12(ha, "enter\n");
6542
6543 if (ep->cm_id) {
6544 qlnxr_iw_issue_event(context,
6545 params,
6546 IW_CM_EVENT_CLOSE,
6547 "IW_CM_EVENT_EVENT_CLOSE");
6548 ep->cm_id->rem_ref(ep->cm_id);
6549 ep->cm_id = NULL;
6550 }
6551
6552 QL_DPRINT12(ha, "exit\n");
6553
6554 return;
6555 }
6556
6557 #if __FreeBSD_version >= 1102000
6558
6559 static void
6560 qlnxr_iw_passive_complete(void *context,
6561 struct ecore_iwarp_cm_event_params *params)
6562 {
6563 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6564 struct qlnxr_dev *dev = ep->dev;
6565 qlnx_host_t *ha;
6566
6567 ha = dev->ha;
6568
6569 /* We will only reach the following state if MPA_REJECT was called on
6570 * passive. In this case there will be no associated QP.
6571 */
6572 if ((params->status == -ECONNREFUSED) && (ep->qp == NULL)) {
6573 QL_DPRINT11(ha, "PASSIVE connection refused releasing ep...\n");
6574 kfree(ep);
6575 return;
6576 }
6577
6578 /* We always issue an established event, however, ofed does not look
6579 * at event code for established. So if there was a failure, we follow
6580 * with close...
6581 */
6582 qlnxr_iw_issue_event(context,
6583 params,
6584 IW_CM_EVENT_ESTABLISHED,
6585 "IW_CM_EVENT_ESTABLISHED");
6586
6587 if (params->status < 0) {
6588 qlnxr_iw_close_event(context, params);
6589 }
6590
6591 return;
6592 }
6593
6594 struct qlnxr_discon_work {
6595 struct work_struct work;
6596 struct qlnxr_iw_ep *ep;
6597 enum ecore_iwarp_event_type event;
6598 int status;
6599 };
6600
6601 static void
6602 qlnxr_iw_disconnect_worker(struct work_struct *work)
6603 {
6604 struct qlnxr_discon_work *dwork =
6605 container_of(work, struct qlnxr_discon_work, work);
6606 struct ecore_rdma_modify_qp_in_params qp_params = { 0 };
6607 struct qlnxr_iw_ep *ep = dwork->ep;
6608 struct qlnxr_dev *dev = ep->dev;
6609 struct qlnxr_qp *qp = ep->qp;
6610 struct iw_cm_event event;
6611
6612 if (qp->destroyed) {
6613 kfree(dwork);
6614 qlnxr_iw_qp_rem_ref(&qp->ibqp);
6615 return;
6616 }
6617
6618 memset(&event, 0, sizeof(event));
6619 event.status = dwork->status;
6620 event.event = IW_CM_EVENT_DISCONNECT;
6621
6622 /* Success means graceful disconnect was requested. modifying
6623 * to SQD is translated to graceful disconnect. O/w reset is sent
6624 */
6625 if (dwork->status)
6626 qp_params.new_state = ECORE_ROCE_QP_STATE_ERR;
6627 else
6628 qp_params.new_state = ECORE_ROCE_QP_STATE_SQD;
6629
6630 kfree(dwork);
6631
6632 if (ep->cm_id)
6633 ep->cm_id->event_handler(ep->cm_id, &event);
6634
6635 SET_FIELD(qp_params.modify_flags,
6636 ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
6637
6638 ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params);
6639
6640 qlnxr_iw_qp_rem_ref(&qp->ibqp);
6641
6642 return;
6643 }
6644
6645 void
6646 qlnxr_iw_disconnect_event(void *context,
6647 struct ecore_iwarp_cm_event_params *params)
6648 {
6649 struct qlnxr_discon_work *work;
6650 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6651 struct qlnxr_dev *dev = ep->dev;
6652 struct qlnxr_qp *qp = ep->qp;
6653
6654 work = kzalloc(sizeof(*work), GFP_ATOMIC);
6655 if (!work)
6656 return;
6657
6658 qlnxr_iw_qp_add_ref(&qp->ibqp);
6659 work->ep = ep;
6660 work->event = params->event;
6661 work->status = params->status;
6662
6663 INIT_WORK(&work->work, qlnxr_iw_disconnect_worker);
6664 queue_work(dev->iwarp_wq, &work->work);
6665
6666 return;
6667 }
6668
6669 #endif /* #if __FreeBSD_version >= 1102000 */
6670
6671 static int
6672 qlnxr_iw_mpa_reply(void *context,
6673 struct ecore_iwarp_cm_event_params *params)
6674 {
6675 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6676 struct qlnxr_dev *dev = ep->dev;
6677 struct ecore_iwarp_send_rtr_in rtr_in;
6678 int rc;
6679 qlnx_host_t *ha;
6680
6681 ha = dev->ha;
6682
6683 QL_DPRINT12(ha, "enter\n");
6684
6685 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
6686 return -EINVAL;
6687
6688 bzero(&rtr_in, sizeof(struct ecore_iwarp_send_rtr_in));
6689 rtr_in.ep_context = params->ep_context;
6690
6691 rc = ecore_iwarp_send_rtr(dev->rdma_ctx, &rtr_in);
6692
6693 QL_DPRINT12(ha, "exit rc = %d\n", rc);
6694 return rc;
6695 }
6696
6697 void
6698 qlnxr_iw_qp_event(void *context,
6699 struct ecore_iwarp_cm_event_params *params,
6700 enum ib_event_type ib_event,
6701 char *str)
6702 {
6703 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6704 struct qlnxr_dev *dev = ep->dev;
6705 struct ib_qp *ibqp = &(ep->qp->ibqp);
6706 struct ib_event event;
6707 qlnx_host_t *ha;
6708
6709 ha = dev->ha;
6710
6711 QL_DPRINT12(ha,
6712 "[context, event, event_handler] = [%p, 0x%x, %s, %p] enter\n",
6713 context, params->event, str, ibqp->event_handler);
6714
6715 if (ibqp->event_handler) {
6716 event.event = ib_event;
6717 event.device = ibqp->device;
6718 event.element.qp = ibqp;
6719 ibqp->event_handler(&event, ibqp->qp_context);
6720 }
6721
6722 return;
6723 }
6724
6725 int
6726 qlnxr_iw_event_handler(void *context,
6727 struct ecore_iwarp_cm_event_params *params)
6728 {
6729 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6730 struct qlnxr_dev *dev = ep->dev;
6731 qlnx_host_t *ha;
6732
6733 ha = dev->ha;
6734
6735 QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] "
6736 "enter\n", context, params->event);
6737
6738 switch (params->event) {
6739 /* Passive side request received */
6740 case ECORE_IWARP_EVENT_MPA_REQUEST:
6741 qlnxr_iw_mpa_request(context, params);
6742 break;
6743
6744 case ECORE_IWARP_EVENT_ACTIVE_MPA_REPLY:
6745 qlnxr_iw_mpa_reply(context, params);
6746 break;
6747
6748 /* Passive side established ( ack on mpa response ) */
6749 case ECORE_IWARP_EVENT_PASSIVE_COMPLETE:
6750
6751 #if __FreeBSD_version >= 1102000
6752
6753 ep->during_connect = 0;
6754 qlnxr_iw_passive_complete(context, params);
6755
6756 #else
6757 qlnxr_iw_issue_event(context,
6758 params,
6759 IW_CM_EVENT_ESTABLISHED,
6760 "IW_CM_EVENT_ESTABLISHED");
6761 #endif /* #if __FreeBSD_version >= 1102000 */
6762 break;
6763
6764 /* Active side reply received */
6765 case ECORE_IWARP_EVENT_ACTIVE_COMPLETE:
6766 ep->during_connect = 0;
6767 qlnxr_iw_issue_event(context,
6768 params,
6769 IW_CM_EVENT_CONNECT_REPLY,
6770 "IW_CM_EVENT_CONNECT_REPLY");
6771 if (params->status < 0) {
6772 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6773
6774 ep->cm_id->rem_ref(ep->cm_id);
6775 ep->cm_id = NULL;
6776 }
6777 break;
6778
6779 case ECORE_IWARP_EVENT_DISCONNECT:
6780
6781 #if __FreeBSD_version >= 1102000
6782 qlnxr_iw_disconnect_event(context, params);
6783 #else
6784 qlnxr_iw_issue_event(context,
6785 params,
6786 IW_CM_EVENT_DISCONNECT,
6787 "IW_CM_EVENT_DISCONNECT");
6788 qlnxr_iw_close_event(context, params);
6789 #endif /* #if __FreeBSD_version >= 1102000 */
6790 break;
6791
6792 case ECORE_IWARP_EVENT_CLOSE:
6793 ep->during_connect = 0;
6794 qlnxr_iw_close_event(context, params);
6795 break;
6796
6797 case ECORE_IWARP_EVENT_RQ_EMPTY:
6798 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6799 "IWARP_EVENT_RQ_EMPTY");
6800 break;
6801
6802 case ECORE_IWARP_EVENT_IRQ_FULL:
6803 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6804 "IWARP_EVENT_IRQ_FULL");
6805 break;
6806
6807 case ECORE_IWARP_EVENT_LLP_TIMEOUT:
6808 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6809 "IWARP_EVENT_LLP_TIMEOUT");
6810 break;
6811
6812 case ECORE_IWARP_EVENT_REMOTE_PROTECTION_ERROR:
6813 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR,
6814 "IWARP_EVENT_REMOTE_PROTECTION_ERROR");
6815 break;
6816
6817 case ECORE_IWARP_EVENT_CQ_OVERFLOW:
6818 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6819 "QED_IWARP_EVENT_CQ_OVERFLOW");
6820 break;
6821
6822 case ECORE_IWARP_EVENT_QP_CATASTROPHIC:
6823 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6824 "QED_IWARP_EVENT_QP_CATASTROPHIC");
6825 break;
6826
6827 case ECORE_IWARP_EVENT_LOCAL_ACCESS_ERROR:
6828 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR,
6829 "IWARP_EVENT_LOCAL_ACCESS_ERROR");
6830 break;
6831
6832 case ECORE_IWARP_EVENT_REMOTE_OPERATION_ERROR:
6833 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6834 "IWARP_EVENT_REMOTE_OPERATION_ERROR");
6835 break;
6836
6837 case ECORE_IWARP_EVENT_TERMINATE_RECEIVED:
6838 QL_DPRINT12(ha, "Got terminate message"
6839 " ECORE_IWARP_EVENT_TERMINATE_RECEIVED\n");
6840 break;
6841
6842 default:
6843 QL_DPRINT12(ha,
6844 "Unknown event [0x%x] received \n", params->event);
6845 break;
6846 };
6847
6848 QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] "
6849 "exit\n", context, params->event);
6850 return 0;
6851 }
6852
6853 static int
6854 qlnxr_addr4_resolve(struct qlnxr_dev *dev,
6855 struct sockaddr_in *src_in,
6856 struct sockaddr_in *dst_in,
6857 u8 *dst_mac)
6858 {
6859 int rc;
6860
6861 #if __FreeBSD_version >= 1100000
6862 rc = arpresolve(dev->ha->ifp, 0, NULL, (struct sockaddr *)dst_in,
6863 dst_mac, NULL, NULL);
6864 #else
6865 struct llentry *lle;
6866
6867 rc = arpresolve(dev->ha->ifp, NULL, NULL, (struct sockaddr *)dst_in,
6868 dst_mac, &lle);
6869 #endif
6870
6871 QL_DPRINT12(dev->ha, "rc = %d "
6872 "sa_len = 0x%x sa_family = 0x%x IP Address = %d.%d.%d.%d "
6873 "Dest MAC %02x:%02x:%02x:%02x:%02x:%02x\n", rc,
6874 dst_in->sin_len, dst_in->sin_family,
6875 NIPQUAD((dst_in->sin_addr.s_addr)),
6876 dst_mac[0], dst_mac[1], dst_mac[2],
6877 dst_mac[3], dst_mac[4], dst_mac[5]);
6878
6879 return rc;
6880 }
6881
6882 int
6883 qlnxr_iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
6884 {
6885 struct qlnxr_dev *dev;
6886 struct ecore_iwarp_connect_out out_params;
6887 struct ecore_iwarp_connect_in in_params;
6888 struct qlnxr_iw_ep *ep;
6889 struct qlnxr_qp *qp;
6890 struct sockaddr_in *laddr;
6891 struct sockaddr_in *raddr;
6892 int rc = 0;
6893 qlnx_host_t *ha;
6894
6895 dev = get_qlnxr_dev((cm_id->device));
6896 ha = dev->ha;
6897
6898 QL_DPRINT12(ha, "[cm_id, conn_param] = [%p, %p] "
6899 "enter \n", cm_id, conn_param);
6900
6901 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
6902 return -EINVAL;
6903
6904 qp = idr_find(&dev->qpidr, conn_param->qpn);
6905
6906 laddr = (struct sockaddr_in *)&cm_id->local_addr;
6907 raddr = (struct sockaddr_in *)&cm_id->remote_addr;
6908
6909 QL_DPRINT12(ha,
6910 "local = [%d.%d.%d.%d, %d] remote = [%d.%d.%d.%d, %d]\n",
6911 NIPQUAD((laddr->sin_addr.s_addr)), laddr->sin_port,
6912 NIPQUAD((raddr->sin_addr.s_addr)), raddr->sin_port);
6913
6914 ep = kzalloc(sizeof(*ep), GFP_KERNEL);
6915 if (!ep) {
6916 QL_DPRINT11(ha, "struct qlnxr_iw_ep "
6917 "alloc memory failed\n");
6918 return -ENOMEM;
6919 }
6920
6921 ep->dev = dev;
6922 ep->qp = qp;
6923 cm_id->add_ref(cm_id);
6924 ep->cm_id = cm_id;
6925
6926 memset(&in_params, 0, sizeof (struct ecore_iwarp_connect_in));
6927 memset(&out_params, 0, sizeof (struct ecore_iwarp_connect_out));
6928
6929 in_params.event_cb = qlnxr_iw_event_handler;
6930 in_params.cb_context = ep;
6931
6932 in_params.cm_info.ip_version = ECORE_TCP_IPV4;
6933
6934 in_params.cm_info.remote_ip[0] = ntohl(raddr->sin_addr.s_addr);
6935 in_params.cm_info.local_ip[0] = ntohl(laddr->sin_addr.s_addr);
6936 in_params.cm_info.remote_port = ntohs(raddr->sin_port);
6937 in_params.cm_info.local_port = ntohs(laddr->sin_port);
6938 in_params.cm_info.vlan = 0;
6939 in_params.mss = dev->ha->ifp->if_mtu - 40;
6940
6941 QL_DPRINT12(ha, "remote_ip = [%d.%d.%d.%d] "
6942 "local_ip = [%d.%d.%d.%d] remote_port = %d local_port = %d "
6943 "vlan = %d\n",
6944 NIPQUAD((in_params.cm_info.remote_ip[0])),
6945 NIPQUAD((in_params.cm_info.local_ip[0])),
6946 in_params.cm_info.remote_port, in_params.cm_info.local_port,
6947 in_params.cm_info.vlan);
6948
6949 rc = qlnxr_addr4_resolve(dev, laddr, raddr, (u8 *)in_params.remote_mac_addr);
6950
6951 if (rc) {
6952 QL_DPRINT11(ha, "qlnxr_addr4_resolve failed\n");
6953 goto err;
6954 }
6955
6956 QL_DPRINT12(ha, "ord = %d ird=%d private_data=%p"
6957 " private_data_len=%d rq_psn=%d\n",
6958 conn_param->ord, conn_param->ird, conn_param->private_data,
6959 conn_param->private_data_len, qp->rq_psn);
6960
6961 in_params.cm_info.ord = conn_param->ord;
6962 in_params.cm_info.ird = conn_param->ird;
6963 in_params.cm_info.private_data = conn_param->private_data;
6964 in_params.cm_info.private_data_len = conn_param->private_data_len;
6965 in_params.qp = qp->ecore_qp;
6966
6967 memcpy(in_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN);
6968
6969 rc = ecore_iwarp_connect(dev->rdma_ctx, &in_params, &out_params);
6970
6971 if (rc) {
6972 QL_DPRINT12(ha, "ecore_iwarp_connect failed\n");
6973 goto err;
6974 }
6975
6976 QL_DPRINT12(ha, "exit\n");
6977
6978 return rc;
6979
6980 err:
6981 cm_id->rem_ref(cm_id);
6982 kfree(ep);
6983
6984 QL_DPRINT12(ha, "exit [%d]\n", rc);
6985 return rc;
6986 }
6987
6988 int
6989 qlnxr_iw_create_listen(struct iw_cm_id *cm_id, int backlog)
6990 {
6991 struct qlnxr_dev *dev;
6992 struct qlnxr_iw_listener *listener;
6993 struct ecore_iwarp_listen_in iparams;
6994 struct ecore_iwarp_listen_out oparams;
6995 struct sockaddr_in *laddr;
6996 qlnx_host_t *ha;
6997 int rc;
6998
6999 dev = get_qlnxr_dev((cm_id->device));
7000 ha = dev->ha;
7001
7002 QL_DPRINT12(ha, "enter\n");
7003
7004 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
7005 return -EINVAL;
7006
7007 laddr = (struct sockaddr_in *)&cm_id->local_addr;
7008
7009 listener = kzalloc(sizeof(*listener), GFP_KERNEL);
7010
7011 if (listener == NULL) {
7012 QL_DPRINT11(ha, "listener memory alloc failed\n");
7013 return -ENOMEM;
7014 }
7015
7016 listener->dev = dev;
7017 cm_id->add_ref(cm_id);
7018 listener->cm_id = cm_id;
7019 listener->backlog = backlog;
7020
7021 memset(&iparams, 0, sizeof (struct ecore_iwarp_listen_in));
7022 memset(&oparams, 0, sizeof (struct ecore_iwarp_listen_out));
7023
7024 iparams.cb_context = listener;
7025 iparams.event_cb = qlnxr_iw_event_handler;
7026 iparams.max_backlog = backlog;
7027
7028 iparams.ip_version = ECORE_TCP_IPV4;
7029
7030 iparams.ip_addr[0] = ntohl(laddr->sin_addr.s_addr);
7031 iparams.port = ntohs(laddr->sin_port);
7032 iparams.vlan = 0;
7033
7034 QL_DPRINT12(ha, "[%d.%d.%d.%d, %d] iparamsport=%d\n",
7035 NIPQUAD((laddr->sin_addr.s_addr)),
7036 laddr->sin_port, iparams.port);
7037
7038 rc = ecore_iwarp_create_listen(dev->rdma_ctx, &iparams, &oparams);
7039 if (rc) {
7040 QL_DPRINT11(ha,
7041 "ecore_iwarp_create_listen failed rc = %d\n", rc);
7042 goto err;
7043 }
7044
7045 listener->ecore_handle = oparams.handle;
7046 cm_id->provider_data = listener;
7047
7048 QL_DPRINT12(ha, "exit\n");
7049 return rc;
7050
7051 err:
7052 cm_id->rem_ref(cm_id);
7053 kfree(listener);
7054
7055 QL_DPRINT12(ha, "exit [%d]\n", rc);
7056 return rc;
7057 }
7058
7059 void
7060 qlnxr_iw_destroy_listen(struct iw_cm_id *cm_id)
7061 {
7062 struct qlnxr_iw_listener *listener = cm_id->provider_data;
7063 struct qlnxr_dev *dev = get_qlnxr_dev((cm_id->device));
7064 int rc = 0;
7065 qlnx_host_t *ha;
7066
7067 ha = dev->ha;
7068
7069 QL_DPRINT12(ha, "enter\n");
7070
7071 if (listener->ecore_handle)
7072 rc = ecore_iwarp_destroy_listen(dev->rdma_ctx,
7073 listener->ecore_handle);
7074
7075 cm_id->rem_ref(cm_id);
7076
7077 QL_DPRINT12(ha, "exit [%d]\n", rc);
7078 return;
7079 }
7080
7081 int
7082 qlnxr_iw_accept(struct iw_cm_id *cm_id,
7083 struct iw_cm_conn_param *conn_param)
7084 {
7085 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data;
7086 struct qlnxr_dev *dev = ep->dev;
7087 struct qlnxr_qp *qp;
7088 struct ecore_iwarp_accept_in params;
7089 int rc;
7090 qlnx_host_t *ha;
7091
7092 ha = dev->ha;
7093
7094 QL_DPRINT12(ha, "enter qpid=%d\n", conn_param->qpn);
7095
7096 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
7097 return -EINVAL;
7098
7099 qp = idr_find(&dev->qpidr, conn_param->qpn);
7100 if (!qp) {
7101 QL_DPRINT11(ha, "idr_find failed invalid qpn = %d\n",
7102 conn_param->qpn);
7103 return -EINVAL;
7104 }
7105 ep->qp = qp;
7106 qp->ep = ep;
7107 cm_id->add_ref(cm_id);
7108 ep->cm_id = cm_id;
7109
7110 params.ep_context = ep->ecore_context;
7111 params.cb_context = ep;
7112 params.qp = ep->qp->ecore_qp;
7113 params.private_data = conn_param->private_data;
7114 params.private_data_len = conn_param->private_data_len;
7115 params.ird = conn_param->ird;
7116 params.ord = conn_param->ord;
7117
7118 rc = ecore_iwarp_accept(dev->rdma_ctx, ¶ms);
7119 if (rc) {
7120 QL_DPRINT11(ha, "ecore_iwarp_accept failed %d\n", rc);
7121 goto err;
7122 }
7123
7124 QL_DPRINT12(ha, "exit\n");
7125 return 0;
7126 err:
7127 cm_id->rem_ref(cm_id);
7128 QL_DPRINT12(ha, "exit rc = %d\n", rc);
7129 return rc;
7130 }
7131
7132 int
7133 qlnxr_iw_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
7134 {
7135 #if __FreeBSD_version >= 1102000
7136
7137 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data;
7138 struct qlnxr_dev *dev = ep->dev;
7139 struct ecore_iwarp_reject_in params;
7140 int rc;
7141
7142 params.ep_context = ep->ecore_context;
7143 params.cb_context = ep;
7144 params.private_data = pdata;
7145 params.private_data_len = pdata_len;
7146 ep->qp = NULL;
7147
7148 rc = ecore_iwarp_reject(dev->rdma_ctx, ¶ms);
7149
7150 return rc;
7151
7152 #else
7153
7154 printf("iWARP reject_cr not implemented\n");
7155 return -EINVAL;
7156
7157 #endif /* #if __FreeBSD_version >= 1102000 */
7158 }
7159
7160 void
7161 qlnxr_iw_qp_add_ref(struct ib_qp *ibqp)
7162 {
7163 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
7164 qlnx_host_t *ha;
7165
7166 ha = qp->dev->ha;
7167
7168 QL_DPRINT12(ha, "enter ibqp = %p\n", ibqp);
7169
7170 atomic_inc(&qp->refcnt);
7171
7172 QL_DPRINT12(ha, "exit \n");
7173 return;
7174 }
7175
7176 void
7177 qlnxr_iw_qp_rem_ref(struct ib_qp *ibqp)
7178 {
7179 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
7180 qlnx_host_t *ha;
7181
7182 ha = qp->dev->ha;
7183
7184 QL_DPRINT12(ha, "enter ibqp = %p qp = %p\n", ibqp, qp);
7185
7186 if (atomic_dec_and_test(&qp->refcnt)) {
7187 qlnxr_idr_remove(qp->dev, qp->qp_id);
7188 kfree(qp);
7189 }
7190
7191 QL_DPRINT12(ha, "exit \n");
7192 return;
7193 }
7194
7195 struct ib_qp *
7196 qlnxr_iw_get_qp(struct ib_device *ibdev, int qpn)
7197 {
7198 struct qlnxr_dev *dev = get_qlnxr_dev(ibdev);
7199 struct ib_qp *qp;
7200 qlnx_host_t *ha;
7201
7202 ha = dev->ha;
7203
7204 QL_DPRINT12(ha, "enter dev = %p ibdev = %p qpn = %d\n", dev, ibdev, qpn);
7205
7206 qp = idr_find(&dev->qpidr, qpn);
7207
7208 QL_DPRINT12(ha, "exit qp = %p\n", qp);
7209
7210 return (qp);
7211 }
7212