1 /*-
2 * Copyright (c) 2013-2020, Mellanox Technologies, Ltd. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26 #ifndef MLX5_IB_H
27 #define MLX5_IB_H
28
29 #include <linux/kernel.h>
30 #include <linux/sched.h>
31 #include <linux/printk.h>
32 #include <linux/netdevice.h>
33 #include <rdma/ib_verbs.h>
34 #include <rdma/ib_smi.h>
35 #include <dev/mlx5/cq.h>
36 #include <dev/mlx5/qp.h>
37 #include <dev/mlx5/srq.h>
38 #include <linux/types.h>
39 #include <dev/mlx5/mlx5_core/transobj.h>
40 #include <rdma/ib_user_verbs.h>
41 #include <rdma/mlx5-abi.h>
42
43 #define mlx5_ib_dbg(dev, format, arg...) \
44 pr_debug("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \
45 __LINE__, current->pid, ##arg)
46
47 #define mlx5_ib_err(dev, format, arg...) \
48 pr_err("%s: ERR: %s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \
49 __LINE__, current->pid, ##arg)
50
51 #define mlx5_ib_warn(dev, format, arg...) \
52 pr_warn("%s: WARN: %s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \
53 __LINE__, current->pid, ##arg)
54
55 #define field_avail(type, fld, sz) (offsetof(type, fld) + \
56 sizeof(((type *)0)->fld) <= (sz))
57 #define MLX5_IB_DEFAULT_UIDX 0xffffff
58 #define MLX5_USER_ASSIGNED_UIDX_MASK __mlx5_mask(qpc, user_index)
59
60 #define MLX5_MKEY_PAGE_SHIFT_MASK __mlx5_mask(mkc, log_page_size)
61
62 enum {
63 MLX5_IB_MMAP_CMD_SHIFT = 8,
64 MLX5_IB_MMAP_CMD_MASK = 0xff,
65 };
66
67 enum mlx5_ib_mmap_cmd {
68 MLX5_IB_MMAP_REGULAR_PAGE = 0,
69 MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES = 1,
70 MLX5_IB_MMAP_WC_PAGE = 2,
71 MLX5_IB_MMAP_NC_PAGE = 3,
72 /* 5 is chosen in order to be compatible with old versions of libmlx5 */
73 MLX5_IB_MMAP_CORE_CLOCK = 5,
74 };
75
76 enum {
77 MLX5_RES_SCAT_DATA32_CQE = 0x1,
78 MLX5_RES_SCAT_DATA64_CQE = 0x2,
79 MLX5_REQ_SCAT_DATA32_CQE = 0x11,
80 MLX5_REQ_SCAT_DATA64_CQE = 0x22,
81 };
82
83 enum mlx5_ib_latency_class {
84 MLX5_IB_LATENCY_CLASS_LOW,
85 MLX5_IB_LATENCY_CLASS_MEDIUM,
86 MLX5_IB_LATENCY_CLASS_HIGH,
87 MLX5_IB_LATENCY_CLASS_FAST_PATH
88 };
89
90 enum mlx5_ib_mad_ifc_flags {
91 MLX5_MAD_IFC_IGNORE_MKEY = 1,
92 MLX5_MAD_IFC_IGNORE_BKEY = 2,
93 MLX5_MAD_IFC_NET_VIEW = 4,
94 };
95
96 enum {
97 MLX5_CROSS_CHANNEL_BFREG = 0,
98 };
99
100 enum {
101 MLX5_CQE_VERSION_V0,
102 MLX5_CQE_VERSION_V1,
103 };
104
105 enum {
106 MLX5_IB_INVALID_UAR_INDEX = BIT(31),
107 MLX5_IB_INVALID_BFREG = BIT(31),
108 };
109
110 struct mlx5_ib_vma_private_data {
111 struct list_head list;
112 struct vm_area_struct *vma;
113 };
114
115 struct mlx5_bfreg_info {
116 u32 *sys_pages;
117 int num_low_latency_bfregs;
118 unsigned int *count;
119
120 /*
121 * protect bfreg allocation data structs
122 */
123 struct mutex lock;
124 u32 ver;
125 u8 lib_uar_4k : 1;
126 u8 lib_uar_dyn : 1;
127 u32 num_sys_pages;
128 u32 num_static_sys_pages;
129 u32 total_num_bfregs;
130 u32 num_dyn_bfregs;
131 };
132
133 struct mlx5_ib_ucontext {
134 struct ib_ucontext ibucontext;
135 struct list_head db_page_list;
136
137 /* protect doorbell record alloc/free
138 */
139 struct mutex db_page_mutex;
140 struct mlx5_bfreg_info bfregi;
141 u8 cqe_version;
142 /* Transport Domain number */
143 u32 tdn;
144 struct list_head vma_private_list;
145 };
146
to_mucontext(struct ib_ucontext * ibucontext)147 static inline struct mlx5_ib_ucontext *to_mucontext(struct ib_ucontext *ibucontext)
148 {
149 return container_of(ibucontext, struct mlx5_ib_ucontext, ibucontext);
150 }
151
152 struct mlx5_ib_pd {
153 struct ib_pd ibpd;
154 u32 pdn;
155 };
156
157 #define MLX5_IB_FLOW_MCAST_PRIO (MLX5_BY_PASS_NUM_PRIOS - 1)
158 #define MLX5_IB_FLOW_LAST_PRIO (MLX5_BY_PASS_NUM_REGULAR_PRIOS - 1)
159 #if (MLX5_IB_FLOW_LAST_PRIO <= 0)
160 #error "Invalid number of bypass priorities"
161 #endif
162 #define MLX5_IB_FLOW_LEFTOVERS_PRIO (MLX5_IB_FLOW_MCAST_PRIO + 1)
163
164 #define MLX5_IB_NUM_FLOW_FT (MLX5_IB_FLOW_LEFTOVERS_PRIO + 1)
165 #define MLX5_IB_NUM_SNIFFER_FTS 2
166 struct mlx5_ib_flow_prio {
167 struct mlx5_flow_table *flow_table;
168 unsigned int refcount;
169 };
170
171 struct mlx5_ib_flow_handler {
172 struct list_head list;
173 struct ib_flow ibflow;
174 struct mlx5_ib_flow_prio *prio;
175 struct mlx5_flow_rule *rule;
176 };
177
178 struct mlx5_ib_flow_db {
179 struct mlx5_ib_flow_prio prios[MLX5_IB_NUM_FLOW_FT];
180 struct mlx5_ib_flow_prio sniffer[MLX5_IB_NUM_SNIFFER_FTS];
181 struct mlx5_flow_table *lag_demux_ft;
182 /* Protect flow steering bypass flow tables
183 * when add/del flow rules.
184 * only single add/removal of flow steering rule could be done
185 * simultaneously.
186 */
187 struct mutex lock;
188 };
189
190 /* Use macros here so that don't have to duplicate
191 * enum ib_send_flags and enum ib_qp_type for low-level driver
192 */
193
194 #define MLX5_IB_SEND_UMR_UNREG IB_SEND_RESERVED_START
195 #define MLX5_IB_SEND_UMR_FAIL_IF_FREE (IB_SEND_RESERVED_START << 1)
196 #define MLX5_IB_SEND_UMR_UPDATE_MTT (IB_SEND_RESERVED_START << 2)
197
198 #define MLX5_IB_SEND_UMR_UPDATE_TRANSLATION (IB_SEND_RESERVED_START << 3)
199 #define MLX5_IB_SEND_UMR_UPDATE_PD (IB_SEND_RESERVED_START << 4)
200 #define MLX5_IB_SEND_UMR_UPDATE_ACCESS IB_SEND_RESERVED_END
201
202 #define MLX5_IB_QPT_REG_UMR IB_QPT_RESERVED1
203 /*
204 * IB_QPT_GSI creates the software wrapper around GSI, and MLX5_IB_QPT_HW_GSI
205 * creates the actual hardware QP.
206 */
207 #define MLX5_IB_QPT_HW_GSI IB_QPT_RESERVED2
208 #define MLX5_IB_QPT_DCI IB_QPT_RESERVED3
209 #define MLX5_IB_QPT_DCT IB_QPT_RESERVED4
210 #define MLX5_IB_WR_UMR IB_WR_RESERVED1
211
212 /* Private QP creation flags to be passed in ib_qp_init_attr.create_flags.
213 *
214 * These flags are intended for internal use by the mlx5_ib driver, and they
215 * rely on the range reserved for that use in the ib_qp_create_flags enum.
216 */
217 #define MLX5_IB_QP_CREATE_SQPN_QP1 IB_QP_CREATE_RESERVED_START
218 #define MLX5_IB_QP_CREATE_WC_TEST (IB_QP_CREATE_RESERVED_START << 1)
219
220 struct wr_list {
221 u16 opcode;
222 u16 next;
223 };
224
225 struct mlx5_ib_wq {
226 u64 *wrid;
227 u32 *wr_data;
228 struct wr_list *w_list;
229 unsigned *wqe_head;
230 u16 unsig_count;
231
232 /* serialize post to the work queue
233 */
234 spinlock_t lock;
235 int wqe_cnt;
236 int max_post;
237 int max_gs;
238 int offset;
239 int wqe_shift;
240 unsigned head;
241 unsigned tail;
242 u16 cur_post;
243 u16 last_poll;
244 void *qend;
245 };
246
247 struct mlx5_ib_rwq {
248 struct ib_wq ibwq;
249 struct mlx5_core_qp core_qp;
250 u32 rq_num_pas;
251 u32 log_rq_stride;
252 u32 log_rq_size;
253 u32 rq_page_offset;
254 u32 log_page_size;
255 struct ib_umem *umem;
256 size_t buf_size;
257 unsigned int page_shift;
258 int create_type;
259 struct mlx5_db db;
260 u32 user_index;
261 u32 wqe_count;
262 u32 wqe_shift;
263 int wq_sig;
264 };
265
266 enum {
267 MLX5_QP_USER,
268 MLX5_QP_KERNEL,
269 MLX5_QP_EMPTY
270 };
271
272 enum {
273 MLX5_WQ_USER,
274 MLX5_WQ_KERNEL
275 };
276
277 struct mlx5_ib_rwq_ind_table {
278 struct ib_rwq_ind_table ib_rwq_ind_tbl;
279 u32 rqtn;
280 };
281
282 /*
283 * Connect-IB can trigger up to four concurrent pagefaults
284 * per-QP.
285 */
286 enum mlx5_ib_pagefault_context {
287 MLX5_IB_PAGEFAULT_RESPONDER_READ,
288 MLX5_IB_PAGEFAULT_REQUESTOR_READ,
289 MLX5_IB_PAGEFAULT_RESPONDER_WRITE,
290 MLX5_IB_PAGEFAULT_REQUESTOR_WRITE,
291 MLX5_IB_PAGEFAULT_CONTEXTS
292 };
293
294 static inline enum mlx5_ib_pagefault_context
mlx5_ib_get_pagefault_context(struct mlx5_pagefault * pagefault)295 mlx5_ib_get_pagefault_context(struct mlx5_pagefault *pagefault)
296 {
297 return pagefault->flags & (MLX5_PFAULT_REQUESTOR | MLX5_PFAULT_WRITE);
298 }
299
300 struct mlx5_ib_pfault {
301 struct work_struct work;
302 struct mlx5_pagefault mpfault;
303 };
304
305 struct mlx5_ib_ubuffer {
306 struct ib_umem *umem;
307 int buf_size;
308 u64 buf_addr;
309 };
310
311 struct mlx5_ib_qp_base {
312 struct mlx5_ib_qp *container_mibqp;
313 struct mlx5_core_qp mqp;
314 struct mlx5_ib_ubuffer ubuffer;
315 };
316
317 struct mlx5_ib_qp_trans {
318 struct mlx5_ib_qp_base base;
319 u16 xrcdn;
320 u8 alt_port;
321 u8 atomic_rd_en;
322 u8 resp_depth;
323 };
324
325 struct mlx5_ib_rss_qp {
326 u32 tirn;
327 };
328
329 struct mlx5_ib_rq {
330 struct mlx5_ib_qp_base base;
331 struct mlx5_ib_wq *rq;
332 struct mlx5_ib_ubuffer ubuffer;
333 struct mlx5_db *doorbell;
334 u32 tirn;
335 u8 state;
336 };
337
338 struct mlx5_ib_sq {
339 struct mlx5_ib_qp_base base;
340 struct mlx5_ib_wq *sq;
341 struct mlx5_ib_ubuffer ubuffer;
342 struct mlx5_db *doorbell;
343 u32 tisn;
344 u8 state;
345 };
346
347 struct mlx5_ib_raw_packet_qp {
348 struct mlx5_ib_sq sq;
349 struct mlx5_ib_rq rq;
350 };
351
352 struct mlx5_bf {
353 int buf_size;
354 unsigned long offset;
355 struct mlx5_sq_bfreg *bfreg;
356 spinlock_t lock32;
357 };
358
359 struct mlx5_ib_qp {
360 struct ib_qp ibqp;
361 union {
362 struct mlx5_ib_qp_trans trans_qp;
363 struct mlx5_ib_raw_packet_qp raw_packet_qp;
364 struct mlx5_ib_rss_qp rss_qp;
365 };
366 struct mlx5_buf buf;
367
368 struct mlx5_db db;
369 struct mlx5_ib_wq rq;
370
371 u8 sq_signal_bits;
372 u8 fm_cache;
373 struct mlx5_ib_wq sq;
374
375 /* serialize qp state modifications
376 */
377 struct mutex mutex;
378 u32 flags;
379 u8 port;
380 u8 state;
381 int wq_sig;
382 int scat_cqe;
383 int max_inline_data;
384 struct mlx5_bf bf;
385 int has_rq;
386
387 /* only for user space QPs. For kernel
388 * we have it from the bf object
389 */
390 int bfregn;
391
392 int create_type;
393
394 /* Store signature errors */
395 bool signature_en;
396
397 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
398 /*
399 * A flag that is true for QP's that are in a state that doesn't
400 * allow page faults, and shouldn't schedule any more faults.
401 */
402 int disable_page_faults;
403 /*
404 * The disable_page_faults_lock protects a QP's disable_page_faults
405 * field, allowing for a thread to atomically check whether the QP
406 * allows page faults, and if so schedule a page fault.
407 */
408 spinlock_t disable_page_faults_lock;
409 struct mlx5_ib_pfault pagefaults[MLX5_IB_PAGEFAULT_CONTEXTS];
410 #endif
411 struct list_head qps_list;
412 struct list_head cq_recv_list;
413 struct list_head cq_send_list;
414 };
415
416 struct mlx5_ib_cq_buf {
417 struct mlx5_buf buf;
418 struct ib_umem *umem;
419 int cqe_size;
420 int nent;
421 };
422
423 enum mlx5_ib_qp_flags {
424 MLX5_IB_QP_LSO = IB_QP_CREATE_IPOIB_UD_LSO,
425 MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK = IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
426 MLX5_IB_QP_CROSS_CHANNEL = IB_QP_CREATE_CROSS_CHANNEL,
427 MLX5_IB_QP_MANAGED_SEND = IB_QP_CREATE_MANAGED_SEND,
428 MLX5_IB_QP_MANAGED_RECV = IB_QP_CREATE_MANAGED_RECV,
429 MLX5_IB_QP_SIGNATURE_HANDLING = 1 << 5,
430 /* QP uses 1 as its source QP number */
431 MLX5_IB_QP_SQPN_QP1 = 1 << 6,
432 MLX5_IB_QP_CAP_SCATTER_FCS = 1 << 7,
433 MLX5_IB_QP_RSS = 1 << 8,
434 };
435
436 struct mlx5_umr_wr {
437 struct ib_send_wr wr;
438 union {
439 u64 virt_addr;
440 u64 offset;
441 } target;
442 struct ib_pd *pd;
443 unsigned int page_shift;
444 unsigned int npages;
445 u32 length;
446 int access_flags;
447 u32 mkey;
448 };
449
umr_wr(const struct ib_send_wr * wr)450 static inline const struct mlx5_umr_wr *umr_wr(const struct ib_send_wr *wr)
451 {
452 return container_of(wr, struct mlx5_umr_wr, wr);
453 }
454
455 struct mlx5_shared_mr_info {
456 int mr_id;
457 struct ib_umem *umem;
458 };
459
460 struct mlx5_ib_cq {
461 struct ib_cq ibcq;
462 struct mlx5_core_cq mcq;
463 struct mlx5_ib_cq_buf buf;
464 struct mlx5_db db;
465
466 /* serialize access to the CQ
467 */
468 spinlock_t lock;
469
470 /* protect resize cq
471 */
472 struct mutex resize_mutex;
473 struct mlx5_ib_cq_buf *resize_buf;
474 struct ib_umem *resize_umem;
475 int cqe_size;
476 struct list_head list_send_qp;
477 struct list_head list_recv_qp;
478 u32 create_flags;
479 struct list_head wc_list;
480 enum ib_cq_notify_flags notify_flags;
481 struct work_struct notify_work;
482 };
483
484 struct mlx5_ib_wc {
485 struct ib_wc wc;
486 struct list_head list;
487 };
488
489 struct mlx5_ib_srq {
490 struct ib_srq ibsrq;
491 struct mlx5_core_srq msrq;
492 struct mlx5_buf buf;
493 struct mlx5_db db;
494 u64 *wrid;
495 /* protect SRQ hanlding
496 */
497 spinlock_t lock;
498 int head;
499 int tail;
500 u16 wqe_ctr;
501 struct ib_umem *umem;
502 /* serialize arming a SRQ
503 */
504 struct mutex mutex;
505 int wq_sig;
506 };
507
508 struct mlx5_ib_xrcd {
509 struct ib_xrcd ibxrcd;
510 u32 xrcdn;
511 };
512
513 enum mlx5_ib_mtt_access_flags {
514 MLX5_IB_MTT_READ = (1 << 0),
515 MLX5_IB_MTT_WRITE = (1 << 1),
516 };
517
518 #define MLX5_IB_MTT_PRESENT (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE)
519
520 struct mlx5_ib_mr {
521 struct ib_mr ibmr;
522 void *descs;
523 dma_addr_t desc_map;
524 int ndescs;
525 int max_descs;
526 int desc_size;
527 int access_mode;
528 struct mlx5_core_mr mmkey;
529 struct ib_umem *umem;
530 struct mlx5_shared_mr_info *smr_info;
531 struct list_head list;
532 int order;
533 int umred;
534 int npages;
535 struct mlx5_ib_dev *dev;
536 u32 out[MLX5_ST_SZ_DW(create_mkey_out)];
537 struct mlx5_core_sig_ctx *sig;
538 int live;
539 void *descs_alloc;
540 int access_flags; /* Needed for rereg MR */
541 struct mlx5_async_work cb_work;
542 };
543
544 struct mlx5_ib_mw {
545 struct ib_mw ibmw;
546 struct mlx5_core_mr mmkey;
547 };
548
549 struct mlx5_ib_umr_context {
550 struct ib_cqe cqe;
551 enum ib_wc_status status;
552 struct completion done;
553 };
554
555 struct umr_common {
556 struct ib_pd *pd;
557 struct ib_cq *cq;
558 struct ib_qp *qp;
559 /* control access to UMR QP
560 */
561 struct semaphore sem;
562 };
563
564 enum {
565 MLX5_FMR_INVALID,
566 MLX5_FMR_VALID,
567 MLX5_FMR_BUSY,
568 };
569
570 struct mlx5_cache_ent {
571 struct list_head head;
572 /* sync access to the cahce entry
573 */
574 spinlock_t lock;
575
576
577 char name[4];
578 u32 order;
579 u32 size;
580 u32 cur;
581 u32 miss;
582 u32 limit;
583
584 struct mlx5_ib_dev *dev;
585 struct work_struct work;
586 struct delayed_work dwork;
587 int pending;
588 };
589
590 struct mlx5_mr_cache {
591 struct workqueue_struct *wq;
592 struct mlx5_cache_ent ent[MAX_MR_CACHE_ENTRIES];
593 int stopped;
594 unsigned long last_add;
595 };
596
597 struct mlx5_ib_gsi_qp;
598
599 struct mlx5_ib_port_resources {
600 struct mlx5_ib_resources *devr;
601 struct mlx5_ib_gsi_qp *gsi;
602 struct work_struct pkey_change_work;
603 };
604
605 struct mlx5_ib_resources {
606 struct ib_cq *c0;
607 struct ib_xrcd *x0;
608 struct ib_xrcd *x1;
609 struct ib_pd *p0;
610 struct ib_srq *s0;
611 struct ib_srq *s1;
612 struct mlx5_ib_port_resources ports[2];
613 /* Protects changes to the port resources */
614 struct mutex mutex;
615 };
616
617 struct mlx5_ib_port {
618 u16 q_cnt_id;
619 };
620
621 struct mlx5_roce {
622 /* Protect mlx5_ib_get_netdev from invoking dev_hold() with a NULL
623 * netdev pointer
624 */
625 rwlock_t netdev_lock;
626 struct ifnet *netdev;
627 struct notifier_block nb;
628 atomic_t next_port;
629 };
630
631 #define MLX5_IB_STATS_COUNT(a,...) a
632 #define MLX5_IB_STATS_VAR(a,b,c,...) b c;
633 #define MLX5_IB_STATS_DESC(a,b,c,d,e,...) d, e,
634
635 #define MLX5_IB_CONG_PARAMS(m) \
636 /* ECN RP */ \
637 m(+1, u64, rp_clamp_tgt_rate, "rp_clamp_tgt_rate", "If set, whenever a CNP is processed, the target rate is updated to be the current rate") \
638 m(+1, u64, rp_clamp_tgt_rate_ati, "rp_clamp_tgt_rate_ati", "If set, when receiving a CNP, the target rate should be updated if the transission rate was increased due to the timer, and not only due to the byte counter") \
639 m(+1, u64, rp_time_reset, "rp_time_reset", "Time in microseconds between rate increases if no CNPs are received") \
640 m(+1, u64, rp_byte_reset, "rp_byte_reset", "Transmitted data in bytes between rate increases if no CNP's are received. A value of zero means disabled.") \
641 m(+1, u64, rp_threshold, "rp_threshold", "The number of times rpByteStage or rpTimeStage can count before the RP rate control state machine advances states") \
642 m(+1, u64, rp_ai_rate, "rp_ai_rate", "The rate, in Mbits per second, used to increase rpTargetRate in the active increase state") \
643 m(+1, u64, rp_hai_rate, "rp_hai_rate", "The rate, in Mbits per second, used to increase rpTargetRate in the hyper increase state") \
644 m(+1, u64, rp_min_dec_fac, "rp_min_dec_fac", "The minimum factor by which the current transmit rate can be changed when processing a CNP. Value is given as a percentage, [1 .. 100]") \
645 m(+1, u64, rp_min_rate, "rp_min_rate", "The minimum value, in Mbps per second, for rate to limit") \
646 m(+1, u64, rp_rate_to_set_on_first_cnp, "rp_rate_to_set_on_first_cnp", "The rate that is set for the flow when a rate limiter is allocated to it upon first CNP received, in Mbps. A value of zero means use full port speed") \
647 m(+1, u64, rp_dce_tcp_g, "rp_dce_tcp_g", "Used to update the congestion estimator, alpha, once every dce_tcp_rtt once every dce_tcp_rtt microseconds") \
648 m(+1, u64, rp_dce_tcp_rtt, "rp_dce_tcp_rtt", "The time between updates of the aolpha value, in microseconds") \
649 m(+1, u64, rp_rate_reduce_monitor_period, "rp_rate_reduce_monitor_period", "The minimum time between two consecutive rate reductions for a single flow") \
650 m(+1, u64, rp_initial_alpha_value, "rp_initial_alpha_value", "The initial value of alpha to use when receiving the first CNP for a flow") \
651 m(+1, u64, rp_gd, "rp_gd", "If a CNP is received, the flow rate is reduced at the beginning of the next rate_reduce_monitor_period interval") \
652 /* ECN NP */ \
653 m(+1, u64, np_cnp_dscp, "np_cnp_dscp", "The DiffServ Code Point of the generated CNP for this port") \
654 m(+1, u64, np_cnp_prio_mode, "np_cnp_prio_mode", "The 802.1p priority value of the generated CNP for this port") \
655 m(+1, u64, np_cnp_prio, "np_cnp_prio", "The 802.1p priority value of the generated CNP for this port")
656
657 #define MLX5_IB_CONG_PARAMS_NUM (0 MLX5_IB_CONG_PARAMS(MLX5_IB_STATS_COUNT))
658
659 #define MLX5_IB_CONG_STATS(m) \
660 m(+1, u64, syndrome, "syndrome", "Syndrome number") \
661 m(+1, u64, rp_cur_flows, "rp_cur_flows", "Number of flows limited") \
662 m(+1, u64, sum_flows, "sum_flows", "Sum of the number of flows limited over time") \
663 m(+1, u64, rp_cnp_ignored, "rp_cnp_ignored", "Number of CNPs and CNMs ignored") \
664 m(+1, u64, rp_cnp_handled, "rp_cnp_handled", "Number of CNPs and CNMs successfully handled") \
665 m(+1, u64, time_stamp, "time_stamp", "Time stamp in microseconds") \
666 m(+1, u64, accumulators_period, "accumulators_period", "The value of X variable for accumulating counters") \
667 m(+1, u64, np_ecn_marked_roce_packets, "np_ecn_marked_roce_packets", "Number of ECN marked packets seen") \
668 m(+1, u64, np_cnp_sent, "np_cnp_sent", "Number of CNPs sent")
669
670 #define MLX5_IB_CONG_STATS_NUM (0 MLX5_IB_CONG_STATS(MLX5_IB_STATS_COUNT))
671
672 #define MLX5_IB_CONG_STATUS(m) \
673 /* ECN RP */ \
674 m(+1, u64, rp_0_enable, "rp_0_enable", "Enable reaction point, priority 0", MLX5_IB_RROCE_ECN_RP, 0, enable) \
675 m(+1, u64, rp_1_enable, "rp_1_enable", "Enable reaction point, priority 1", MLX5_IB_RROCE_ECN_RP, 1, enable) \
676 m(+1, u64, rp_2_enable, "rp_2_enable", "Enable reaction point, priority 2", MLX5_IB_RROCE_ECN_RP, 2, enable) \
677 m(+1, u64, rp_3_enable, "rp_3_enable", "Enable reaction point, priority 3", MLX5_IB_RROCE_ECN_RP, 3, enable) \
678 m(+1, u64, rp_4_enable, "rp_4_enable", "Enable reaction point, priority 4", MLX5_IB_RROCE_ECN_RP, 4, enable) \
679 m(+1, u64, rp_5_enable, "rp_5_enable", "Enable reaction point, priority 5", MLX5_IB_RROCE_ECN_RP, 5, enable) \
680 m(+1, u64, rp_6_enable, "rp_6_enable", "Enable reaction point, priority 6", MLX5_IB_RROCE_ECN_RP, 6, enable) \
681 m(+1, u64, rp_7_enable, "rp_7_enable", "Enable reaction point, priority 7", MLX5_IB_RROCE_ECN_RP, 7, enable) \
682 m(+1, u64, rp_8_enable, "rp_8_enable", "Enable reaction point, priority 8", MLX5_IB_RROCE_ECN_RP, 8, enable) \
683 m(+1, u64, rp_9_enable, "rp_9_enable", "Enable reaction point, priority 9", MLX5_IB_RROCE_ECN_RP, 9, enable) \
684 m(+1, u64, rp_10_enable, "rp_10_enable", "Enable reaction point, priority 10", MLX5_IB_RROCE_ECN_RP, 10, enable) \
685 m(+1, u64, rp_11_enable, "rp_11_enable", "Enable reaction point, priority 11", MLX5_IB_RROCE_ECN_RP, 11, enable) \
686 m(+1, u64, rp_12_enable, "rp_12_enable", "Enable reaction point, priority 12", MLX5_IB_RROCE_ECN_RP, 12, enable) \
687 m(+1, u64, rp_13_enable, "rp_13_enable", "Enable reaction point, priority 13", MLX5_IB_RROCE_ECN_RP, 13, enable) \
688 m(+1, u64, rp_14_enable, "rp_14_enable", "Enable reaction point, priority 14", MLX5_IB_RROCE_ECN_RP, 14, enable) \
689 m(+1, u64, rp_15_enable, "rp_15_enable", "Enable reaction point, priority 15", MLX5_IB_RROCE_ECN_RP, 15, enable) \
690 /* ECN NP */ \
691 m(+1, u64, np_0_enable, "np_0_enable", "Enable notification point, priority 0", MLX5_IB_RROCE_ECN_NP, 0, enable) \
692 m(+1, u64, np_1_enable, "np_1_enable", "Enable notification point, priority 1", MLX5_IB_RROCE_ECN_NP, 1, enable) \
693 m(+1, u64, np_2_enable, "np_2_enable", "Enable notification point, priority 2", MLX5_IB_RROCE_ECN_NP, 2, enable) \
694 m(+1, u64, np_3_enable, "np_3_enable", "Enable notification point, priority 3", MLX5_IB_RROCE_ECN_NP, 3, enable) \
695 m(+1, u64, np_4_enable, "np_4_enable", "Enable notification point, priority 4", MLX5_IB_RROCE_ECN_NP, 4, enable) \
696 m(+1, u64, np_5_enable, "np_5_enable", "Enable notification point, priority 5", MLX5_IB_RROCE_ECN_NP, 5, enable) \
697 m(+1, u64, np_6_enable, "np_6_enable", "Enable notification point, priority 6", MLX5_IB_RROCE_ECN_NP, 6, enable) \
698 m(+1, u64, np_7_enable, "np_7_enable", "Enable notification point, priority 7", MLX5_IB_RROCE_ECN_NP, 7, enable) \
699 m(+1, u64, np_8_enable, "np_8_enable", "Enable notification point, priority 8", MLX5_IB_RROCE_ECN_NP, 8, enable) \
700 m(+1, u64, np_9_enable, "np_9_enable", "Enable notification point, priority 9", MLX5_IB_RROCE_ECN_NP, 9, enable) \
701 m(+1, u64, np_10_enable, "np_10_enable", "Enable notification point, priority 10", MLX5_IB_RROCE_ECN_NP, 10, enable) \
702 m(+1, u64, np_11_enable, "np_11_enable", "Enable notification point, priority 11", MLX5_IB_RROCE_ECN_NP, 11, enable) \
703 m(+1, u64, np_12_enable, "np_12_enable", "Enable notification point, priority 12", MLX5_IB_RROCE_ECN_NP, 12, enable) \
704 m(+1, u64, np_13_enable, "np_13_enable", "Enable notification point, priority 13", MLX5_IB_RROCE_ECN_NP, 13, enable) \
705 m(+1, u64, np_14_enable, "np_14_enable", "Enable notification point, priority 14", MLX5_IB_RROCE_ECN_NP, 14, enable) \
706 m(+1, u64, np_15_enable, "np_15_enable", "Enable notification point, priority 15", MLX5_IB_RROCE_ECN_NP, 15, enable) \
707
708 #define MLX5_IB_CONG_STATUS_NUM (0 MLX5_IB_CONG_STATUS(MLX5_IB_STATS_COUNT))
709
710 struct mlx5_ib_congestion {
711 struct sysctl_ctx_list ctx;
712 struct sx lock;
713 struct delayed_work dwork;
714 union {
715 u64 arg[1];
716 struct {
717 MLX5_IB_CONG_PARAMS(MLX5_IB_STATS_VAR)
718 MLX5_IB_CONG_STATS(MLX5_IB_STATS_VAR)
719 MLX5_IB_CONG_STATUS(MLX5_IB_STATS_VAR)
720 };
721 };
722 };
723
724 struct mlx5_ib_dev {
725 struct ib_device ib_dev;
726 struct mlx5_core_dev *mdev;
727 struct mlx5_roce roce;
728 MLX5_DECLARE_DOORBELL_LOCK(uar_lock);
729 int num_ports;
730 /* serialize update of capability mask
731 */
732 struct mutex cap_mask_mutex;
733 bool ib_active;
734 struct umr_common umrc;
735 /* sync used page count stats
736 */
737 struct mlx5_ib_resources devr;
738 struct mlx5_mr_cache cache;
739 struct timer_list delay_timer;
740 /* Prevents soft lock on massive reg MRs */
741 struct mutex slow_path_mutex;
742 int fill_delay;
743 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
744 struct ib_odp_caps odp_caps;
745 /*
746 * Sleepable RCU that prevents destruction of MRs while they are still
747 * being used by a page fault handler.
748 */
749 struct srcu_struct mr_srcu;
750 #endif
751 struct mlx5_ib_flow_db flow_db;
752 /* protect resources needed as part of reset flow */
753 spinlock_t reset_flow_resource_lock;
754 struct list_head qp_list;
755 /* Array with num_ports elements */
756 struct mlx5_ib_port *port;
757 struct mlx5_sq_bfreg bfreg;
758 struct mlx5_sq_bfreg wc_bfreg;
759 struct mlx5_sq_bfreg fp_bfreg;
760 struct mlx5_ib_congestion congestion;
761
762 struct mlx5_async_ctx async_ctx;
763
764 /* protect the user_td */
765 struct mutex lb_mutex;
766 u32 user_td;
767 };
768
to_mibcq(struct mlx5_core_cq * mcq)769 static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
770 {
771 return container_of(mcq, struct mlx5_ib_cq, mcq);
772 }
773
to_mxrcd(struct ib_xrcd * ibxrcd)774 static inline struct mlx5_ib_xrcd *to_mxrcd(struct ib_xrcd *ibxrcd)
775 {
776 return container_of(ibxrcd, struct mlx5_ib_xrcd, ibxrcd);
777 }
778
to_mdev(struct ib_device * ibdev)779 static inline struct mlx5_ib_dev *to_mdev(struct ib_device *ibdev)
780 {
781 return container_of(ibdev, struct mlx5_ib_dev, ib_dev);
782 }
783
to_mcq(struct ib_cq * ibcq)784 static inline struct mlx5_ib_cq *to_mcq(struct ib_cq *ibcq)
785 {
786 return container_of(ibcq, struct mlx5_ib_cq, ibcq);
787 }
788
to_mibqp(struct mlx5_core_qp * mqp)789 static inline struct mlx5_ib_qp *to_mibqp(struct mlx5_core_qp *mqp)
790 {
791 return container_of(mqp, struct mlx5_ib_qp_base, mqp)->container_mibqp;
792 }
793
to_mibrwq(struct mlx5_core_qp * core_qp)794 static inline struct mlx5_ib_rwq *to_mibrwq(struct mlx5_core_qp *core_qp)
795 {
796 return container_of(core_qp, struct mlx5_ib_rwq, core_qp);
797 }
798
to_mibmr(struct mlx5_core_mr * mmkey)799 static inline struct mlx5_ib_mr *to_mibmr(struct mlx5_core_mr *mmkey)
800 {
801 return container_of(mmkey, struct mlx5_ib_mr, mmkey);
802 }
803
to_mpd(struct ib_pd * ibpd)804 static inline struct mlx5_ib_pd *to_mpd(struct ib_pd *ibpd)
805 {
806 return container_of(ibpd, struct mlx5_ib_pd, ibpd);
807 }
808
to_msrq(struct ib_srq * ibsrq)809 static inline struct mlx5_ib_srq *to_msrq(struct ib_srq *ibsrq)
810 {
811 return container_of(ibsrq, struct mlx5_ib_srq, ibsrq);
812 }
813
to_mqp(struct ib_qp * ibqp)814 static inline struct mlx5_ib_qp *to_mqp(struct ib_qp *ibqp)
815 {
816 return container_of(ibqp, struct mlx5_ib_qp, ibqp);
817 }
818
to_mrwq(struct ib_wq * ibwq)819 static inline struct mlx5_ib_rwq *to_mrwq(struct ib_wq *ibwq)
820 {
821 return container_of(ibwq, struct mlx5_ib_rwq, ibwq);
822 }
823
to_mrwq_ind_table(struct ib_rwq_ind_table * ib_rwq_ind_tbl)824 static inline struct mlx5_ib_rwq_ind_table *to_mrwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
825 {
826 return container_of(ib_rwq_ind_tbl, struct mlx5_ib_rwq_ind_table, ib_rwq_ind_tbl);
827 }
828
to_mibsrq(struct mlx5_core_srq * msrq)829 static inline struct mlx5_ib_srq *to_mibsrq(struct mlx5_core_srq *msrq)
830 {
831 return container_of(msrq, struct mlx5_ib_srq, msrq);
832 }
833
to_mmr(struct ib_mr * ibmr)834 static inline struct mlx5_ib_mr *to_mmr(struct ib_mr *ibmr)
835 {
836 return container_of(ibmr, struct mlx5_ib_mr, ibmr);
837 }
838
to_mmw(struct ib_mw * ibmw)839 static inline struct mlx5_ib_mw *to_mmw(struct ib_mw *ibmw)
840 {
841 return container_of(ibmw, struct mlx5_ib_mw, ibmw);
842 }
843
844 struct mlx5_ib_ah {
845 struct ib_ah ibah;
846 struct mlx5_av av;
847 };
848
to_mah(struct ib_ah * ibah)849 static inline struct mlx5_ib_ah *to_mah(struct ib_ah *ibah)
850 {
851 return container_of(ibah, struct mlx5_ib_ah, ibah);
852 }
853
854 int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt,
855 struct mlx5_db *db);
856 void mlx5_ib_db_unmap_user(struct mlx5_ib_ucontext *context, struct mlx5_db *db);
857 void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);
858 void mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);
859 void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index);
860 int mlx5_MAD_IFC(struct mlx5_ib_dev *dev, int ignore_mkey, int ignore_bkey,
861 u8 port, const struct ib_wc *in_wc, const struct ib_grh *in_grh,
862 const void *in_mad, void *response_mad);
863 struct ib_ah *mlx5_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr,
864 struct ib_udata *udata);
865 int mlx5_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr);
866 int mlx5_ib_destroy_ah(struct ib_ah *ah);
867 struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
868 struct ib_srq_init_attr *init_attr,
869 struct ib_udata *udata);
870 int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
871 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
872 int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr);
873 int mlx5_ib_destroy_srq(struct ib_srq *srq);
874 int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
875 const struct ib_recv_wr **bad_wr);
876 struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,
877 struct ib_qp_init_attr *init_attr,
878 struct ib_udata *udata);
879 int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
880 int attr_mask, struct ib_udata *udata);
881 int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
882 struct ib_qp_init_attr *qp_init_attr);
883 int mlx5_ib_destroy_qp(struct ib_qp *qp);
884 int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
885 const struct ib_send_wr **bad_wr);
886 int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
887 const struct ib_recv_wr **bad_wr);
888 void *mlx5_get_send_wqe(struct mlx5_ib_qp *qp, int n);
889 int mlx5_ib_read_user_wqe(struct mlx5_ib_qp *qp, int send, int wqe_index,
890 void *buffer, u32 length,
891 struct mlx5_ib_qp_base *base);
892 struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
893 const struct ib_cq_init_attr *attr,
894 struct ib_ucontext *context,
895 struct ib_udata *udata);
896 int mlx5_ib_destroy_cq(struct ib_cq *cq);
897 int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
898 int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
899 int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);
900 int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata);
901 struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc);
902 struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
903 u64 virt_addr, int access_flags,
904 struct ib_udata *udata);
905 struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
906 struct ib_udata *udata);
907 int mlx5_ib_dealloc_mw(struct ib_mw *mw);
908 int mlx5_ib_update_mtt(struct mlx5_ib_mr *mr, u64 start_page_index,
909 int npages, int zap);
910 int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
911 u64 length, u64 virt_addr, int access_flags,
912 struct ib_pd *pd, struct ib_udata *udata);
913 int mlx5_ib_dereg_mr(struct ib_mr *ibmr);
914 struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
915 enum ib_mr_type mr_type,
916 u32 max_num_sg);
917 int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
918 unsigned int *sg_offset);
919 int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
920 const struct ib_wc *in_wc, const struct ib_grh *in_grh,
921 const struct ib_mad_hdr *in, size_t in_mad_size,
922 struct ib_mad_hdr *out, size_t *out_mad_size,
923 u16 *out_mad_pkey_index);
924 struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,
925 struct ib_ucontext *context,
926 struct ib_udata *udata);
927 int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd);
928 int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset);
929 int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port);
930 int mlx5_query_mad_ifc_smp_attr_node_info(struct ib_device *ibdev,
931 struct ib_smp *out_mad);
932 int mlx5_query_mad_ifc_system_image_guid(struct ib_device *ibdev,
933 __be64 *sys_image_guid);
934 int mlx5_query_mad_ifc_max_pkeys(struct ib_device *ibdev,
935 u16 *max_pkeys);
936 int mlx5_query_mad_ifc_vendor_id(struct ib_device *ibdev,
937 u32 *vendor_id);
938 int mlx5_query_mad_ifc_node_desc(struct mlx5_ib_dev *dev, char *node_desc);
939 int mlx5_query_mad_ifc_node_guid(struct mlx5_ib_dev *dev, __be64 *node_guid);
940 int mlx5_query_mad_ifc_pkey(struct ib_device *ibdev, u8 port, u16 index,
941 u16 *pkey);
942 int mlx5_query_mad_ifc_gids(struct ib_device *ibdev, u8 port, int index,
943 union ib_gid *gid);
944 int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port,
945 struct ib_port_attr *props);
946 int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
947 struct ib_port_attr *props);
948 int mlx5_ib_init_fmr(struct mlx5_ib_dev *dev);
949 void mlx5_ib_cleanup_fmr(struct mlx5_ib_dev *dev);
950 void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr,
951 unsigned long max_page_shift,
952 int *count, int *shift,
953 int *ncont, int *order);
954 void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
955 int page_shift, size_t offset, size_t num_pages,
956 __be64 *pas, int access_flags);
957 void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
958 int page_shift, __be64 *pas, int access_flags);
959 void mlx5_ib_copy_pas(u64 *old, u64 *new, int step, int num);
960 int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq);
961 int mlx5_mr_cache_init(struct mlx5_ib_dev *dev);
962 int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev);
963 int mlx5_mr_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift);
964 int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
965 struct ib_mr_status *mr_status);
966 struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
967 struct ib_wq_init_attr *init_attr,
968 struct ib_udata *udata);
969 int mlx5_ib_destroy_wq(struct ib_wq *wq);
970 int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
971 u32 wq_attr_mask, struct ib_udata *udata);
972 struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,
973 struct ib_rwq_ind_table_init_attr *init_attr,
974 struct ib_udata *udata);
975 int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);
976
977 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
978 extern struct workqueue_struct *mlx5_ib_page_fault_wq;
979
980 void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev);
981 void mlx5_ib_mr_pfault_handler(struct mlx5_ib_qp *qp,
982 struct mlx5_ib_pfault *pfault);
983 void mlx5_ib_odp_create_qp(struct mlx5_ib_qp *qp);
984 int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev);
985 void mlx5_ib_odp_remove_one(struct mlx5_ib_dev *ibdev);
986 int __init mlx5_ib_odp_init(void);
987 void mlx5_ib_odp_cleanup(void);
988 void mlx5_ib_qp_disable_pagefaults(struct mlx5_ib_qp *qp);
989 void mlx5_ib_qp_enable_pagefaults(struct mlx5_ib_qp *qp);
990 void mlx5_ib_invalidate_range(struct ib_umem *umem, unsigned long start,
991 unsigned long end);
992 #else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev * dev)993 static inline void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev)
994 {
995 return;
996 }
997
mlx5_ib_odp_create_qp(struct mlx5_ib_qp * qp)998 static inline void mlx5_ib_odp_create_qp(struct mlx5_ib_qp *qp) {}
mlx5_ib_odp_init_one(struct mlx5_ib_dev * ibdev)999 static inline int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev) { return 0; }
mlx5_ib_odp_remove_one(struct mlx5_ib_dev * ibdev)1000 static inline void mlx5_ib_odp_remove_one(struct mlx5_ib_dev *ibdev) {}
mlx5_ib_odp_init(void)1001 static inline int mlx5_ib_odp_init(void) { return 0; }
mlx5_ib_odp_cleanup(void)1002 static inline void mlx5_ib_odp_cleanup(void) {}
mlx5_ib_qp_disable_pagefaults(struct mlx5_ib_qp * qp)1003 static inline void mlx5_ib_qp_disable_pagefaults(struct mlx5_ib_qp *qp) {}
mlx5_ib_qp_enable_pagefaults(struct mlx5_ib_qp * qp)1004 static inline void mlx5_ib_qp_enable_pagefaults(struct mlx5_ib_qp *qp) {}
1005
1006 #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
1007
1008 int mlx5_ib_get_vf_config(struct ib_device *device, int vf,
1009 u8 port, struct ifla_vf_info *info);
1010 int mlx5_ib_set_vf_link_state(struct ib_device *device, int vf,
1011 u8 port, int state);
1012 int mlx5_ib_get_vf_stats(struct ib_device *device, int vf,
1013 u8 port, struct ifla_vf_stats *stats);
1014 int mlx5_ib_set_vf_guid(struct ib_device *device, int vf, u8 port,
1015 u64 guid, int type);
1016
1017 __be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev, u8 port_num,
1018 int index);
1019 int mlx5_get_roce_gid_type(struct mlx5_ib_dev *dev, u8 port_num,
1020 int index, enum ib_gid_type *gid_type);
1021
1022 /* GSI QP helper functions */
1023 struct ib_qp *mlx5_ib_gsi_create_qp(struct ib_pd *pd,
1024 struct ib_qp_init_attr *init_attr);
1025 int mlx5_ib_gsi_destroy_qp(struct ib_qp *qp);
1026 int mlx5_ib_gsi_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,
1027 int attr_mask);
1028 int mlx5_ib_gsi_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
1029 int qp_attr_mask,
1030 struct ib_qp_init_attr *qp_init_attr);
1031 int mlx5_ib_gsi_post_send(struct ib_qp *qp, const struct ib_send_wr *wr,
1032 const struct ib_send_wr **bad_wr);
1033 int mlx5_ib_gsi_post_recv(struct ib_qp *qp, const struct ib_recv_wr *wr,
1034 const struct ib_recv_wr **bad_wr);
1035 void mlx5_ib_gsi_pkey_change(struct mlx5_ib_gsi_qp *gsi);
1036
1037 int mlx5_ib_generate_wc(struct ib_cq *ibcq, struct ib_wc *wc);
1038
1039 void mlx5_ib_free_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi,
1040 int bfregn);
init_query_mad(struct ib_smp * mad)1041 static inline void init_query_mad(struct ib_smp *mad)
1042 {
1043 mad->base_version = 1;
1044 mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
1045 mad->class_version = 1;
1046 mad->method = IB_MGMT_METHOD_GET;
1047 }
1048
convert_access(int acc)1049 static inline u8 convert_access(int acc)
1050 {
1051 return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC : 0) |
1052 (acc & IB_ACCESS_REMOTE_WRITE ? MLX5_PERM_REMOTE_WRITE : 0) |
1053 (acc & IB_ACCESS_REMOTE_READ ? MLX5_PERM_REMOTE_READ : 0) |
1054 (acc & IB_ACCESS_LOCAL_WRITE ? MLX5_PERM_LOCAL_WRITE : 0) |
1055 MLX5_PERM_LOCAL_READ;
1056 }
1057
is_qp1(enum ib_qp_type qp_type)1058 static inline int is_qp1(enum ib_qp_type qp_type)
1059 {
1060 return qp_type == MLX5_IB_QPT_HW_GSI;
1061 }
1062
1063 #define MLX5_MAX_UMR_SHIFT 16
1064 #define MLX5_MAX_UMR_PAGES (1 << MLX5_MAX_UMR_SHIFT)
1065
check_cq_create_flags(u32 flags)1066 static inline u32 check_cq_create_flags(u32 flags)
1067 {
1068 /*
1069 * It returns non-zero value for unsupported CQ
1070 * create flags, otherwise it returns zero.
1071 */
1072 return (flags & ~(IB_CQ_FLAGS_IGNORE_OVERRUN |
1073 IB_CQ_FLAGS_TIMESTAMP_COMPLETION));
1074 }
1075
verify_assign_uidx(u8 cqe_version,u32 cmd_uidx,u32 * user_index)1076 static inline int verify_assign_uidx(u8 cqe_version, u32 cmd_uidx,
1077 u32 *user_index)
1078 {
1079 if (cqe_version) {
1080 if ((cmd_uidx == MLX5_IB_DEFAULT_UIDX) ||
1081 (cmd_uidx & ~MLX5_USER_ASSIGNED_UIDX_MASK))
1082 return -EINVAL;
1083 *user_index = cmd_uidx;
1084 } else {
1085 *user_index = MLX5_IB_DEFAULT_UIDX;
1086 }
1087
1088 return 0;
1089 }
1090
get_qp_user_index(struct mlx5_ib_ucontext * ucontext,struct mlx5_ib_create_qp * ucmd,int inlen,u32 * user_index)1091 static inline int get_qp_user_index(struct mlx5_ib_ucontext *ucontext,
1092 struct mlx5_ib_create_qp *ucmd,
1093 int inlen,
1094 u32 *user_index)
1095 {
1096 u8 cqe_version = ucontext->cqe_version;
1097
1098 if (field_avail(struct mlx5_ib_create_qp, uidx, inlen) &&
1099 !cqe_version && (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
1100 return 0;
1101
1102 if (!!(field_avail(struct mlx5_ib_create_qp, uidx, inlen) !=
1103 !!cqe_version))
1104 return -EINVAL;
1105
1106 return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);
1107 }
1108
get_srq_user_index(struct mlx5_ib_ucontext * ucontext,struct mlx5_ib_create_srq * ucmd,int inlen,u32 * user_index)1109 static inline int get_srq_user_index(struct mlx5_ib_ucontext *ucontext,
1110 struct mlx5_ib_create_srq *ucmd,
1111 int inlen,
1112 u32 *user_index)
1113 {
1114 u8 cqe_version = ucontext->cqe_version;
1115
1116 if (field_avail(struct mlx5_ib_create_srq, uidx, inlen) &&
1117 !cqe_version && (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
1118 return 0;
1119
1120 if (!!(field_avail(struct mlx5_ib_create_srq, uidx, inlen) !=
1121 !!cqe_version))
1122 return -EINVAL;
1123
1124 return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);
1125 }
1126
1127 void mlx5_ib_cleanup_congestion(struct mlx5_ib_dev *);
1128 int mlx5_ib_init_congestion(struct mlx5_ib_dev *);
1129
get_uars_per_sys_page(struct mlx5_ib_dev * dev,bool lib_support)1130 static inline int get_uars_per_sys_page(struct mlx5_ib_dev *dev, bool lib_support)
1131 {
1132 return lib_support && MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1133 MLX5_UARS_IN_PAGE : 1;
1134 }
1135
get_num_static_uars(struct mlx5_ib_dev * dev,struct mlx5_bfreg_info * bfregi)1136 static inline int get_num_static_uars(struct mlx5_ib_dev *dev,
1137 struct mlx5_bfreg_info *bfregi)
1138 {
1139 return get_uars_per_sys_page(dev, bfregi->lib_uar_4k) * bfregi->num_static_sys_pages;
1140 }
1141
1142 int bfregn_to_uar_index(struct mlx5_ib_dev *dev,
1143 struct mlx5_bfreg_info *bfregi, u32 bfregn,
1144 bool dyn_bfreg);
1145
1146 #endif /* MLX5_IB_H */
1147