1 /*-
2 * Copyright (c) 2019-2021 Mellanox Technologies. All rights reserved.
3 * Copyright (c) 2022 NVIDIA corporation & affiliates.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 #include "opt_kern_tls.h"
28 #include "opt_rss.h"
29 #include "opt_ratelimit.h"
30
31 #include <dev/mlx5/mlx5_en/en.h>
32
33 #include <dev/mlx5/tls.h>
34
35 #include <linux/delay.h>
36 #include <sys/ktls.h>
37 #include <opencrypto/cryptodev.h>
38
39 #ifdef KERN_TLS
40
41 MALLOC_DEFINE(M_MLX5E_TLS, "MLX5E_TLS", "MLX5 ethernet HW TLS");
42
43 /* software TLS context */
44 struct mlx5_ifc_sw_tls_cntx_bits {
45 struct mlx5_ifc_tls_static_params_bits param;
46 struct mlx5_ifc_tls_progress_params_bits progress;
47 struct {
48 uint8_t key_data[8][0x20];
49 uint8_t key_len[0x20];
50 } key;
51 };
52
53 CTASSERT(MLX5_ST_SZ_BYTES(sw_tls_cntx) <= sizeof(((struct mlx5e_tls_tag *)0)->crypto_params));
54 CTASSERT(MLX5_ST_SZ_BYTES(mkc) == sizeof(((struct mlx5e_tx_umr_wqe *)0)->mkc));
55
56 static const char *mlx5e_tls_stats_desc[] = {
57 MLX5E_TLS_STATS(MLX5E_STATS_DESC)
58 };
59
60 static void mlx5e_tls_work(struct work_struct *);
61
62 static int
mlx5e_tls_tag_import(void * arg,void ** store,int cnt,int domain,int flags)63 mlx5e_tls_tag_import(void *arg, void **store, int cnt, int domain, int flags)
64 {
65 struct mlx5e_tls_tag *ptag;
66 int i;
67
68 for (i = 0; i != cnt; i++) {
69 ptag = malloc_domainset(sizeof(*ptag), M_MLX5E_TLS,
70 mlx5_dev_domainset(arg), flags | M_ZERO);
71 mtx_init(&ptag->mtx, "mlx5-tls-tag-mtx", NULL, MTX_DEF);
72 INIT_WORK(&ptag->work, mlx5e_tls_work);
73 store[i] = ptag;
74 }
75 return (i);
76 }
77
78 static void
mlx5e_tls_tag_release(void * arg,void ** store,int cnt)79 mlx5e_tls_tag_release(void *arg, void **store, int cnt)
80 {
81 struct mlx5e_tls_tag *ptag;
82 struct mlx5e_priv *priv;
83 struct mlx5e_tls *ptls;
84 int i;
85
86 for (i = 0; i != cnt; i++) {
87 ptag = store[i];
88 ptls = ptag->tls;
89 priv = container_of(ptls, struct mlx5e_priv, tls);
90
91 flush_work(&ptag->work);
92
93 if (ptag->tisn != 0) {
94 mlx5_tls_close_tis(priv->mdev, ptag->tisn);
95 atomic_add_32(&ptls->num_resources, -1U);
96 }
97
98 mtx_destroy(&ptag->mtx);
99
100 free(ptag, M_MLX5E_TLS);
101 }
102 }
103
104 static void
mlx5e_tls_tag_zfree(struct mlx5e_tls_tag * ptag)105 mlx5e_tls_tag_zfree(struct mlx5e_tls_tag *ptag)
106 {
107 /* make sure any unhandled taskqueue events are ignored */
108 ptag->state = MLX5E_TLS_ST_FREED;
109
110 /* reset some variables */
111 ptag->dek_index = 0;
112 ptag->dek_index_ok = 0;
113
114 /* avoid leaking keys */
115 memset(ptag->crypto_params, 0, sizeof(ptag->crypto_params));
116
117 /* update number of TIS contexts */
118 if (ptag->tisn == 0)
119 atomic_add_32(&ptag->tls->num_resources, -1U);
120
121 /* return tag to UMA */
122 uma_zfree(ptag->tls->zone, ptag);
123 }
124
125 int
mlx5e_tls_init(struct mlx5e_priv * priv)126 mlx5e_tls_init(struct mlx5e_priv *priv)
127 {
128 struct mlx5e_tls *ptls = &priv->tls;
129 struct sysctl_oid *node;
130 uint32_t x;
131
132 if (MLX5_CAP_GEN(priv->mdev, tls_tx) == 0 ||
133 MLX5_CAP_GEN(priv->mdev, log_max_dek) == 0)
134 return (0);
135
136 ptls->wq = create_singlethread_workqueue("mlx5-tls-wq");
137 if (ptls->wq == NULL)
138 return (ENOMEM);
139
140 sysctl_ctx_init(&ptls->ctx);
141
142 snprintf(ptls->zname, sizeof(ptls->zname),
143 "mlx5_%u_tls", device_get_unit(priv->mdev->pdev->dev.bsddev));
144
145 ptls->zone = uma_zcache_create(ptls->zname,
146 sizeof(struct mlx5e_tls_tag), NULL, NULL, NULL, NULL,
147 mlx5e_tls_tag_import, mlx5e_tls_tag_release, priv->mdev,
148 UMA_ZONE_UNMANAGED);
149
150 /* shared between RX and TX TLS */
151 ptls->max_resources = 1U << (MLX5_CAP_GEN(priv->mdev, log_max_dek) - 1);
152
153 for (x = 0; x != MLX5E_TLS_STATS_NUM; x++)
154 ptls->stats.arg[x] = counter_u64_alloc(M_WAITOK);
155
156 ptls->init = 1;
157
158 node = SYSCTL_ADD_NODE(&priv->sysctl_ctx,
159 SYSCTL_CHILDREN(priv->sysctl_ifnet), OID_AUTO,
160 "tls", CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, "Hardware TLS offload");
161 if (node == NULL)
162 return (0);
163
164 mlx5e_create_counter_stats(&ptls->ctx,
165 SYSCTL_CHILDREN(node), "stats",
166 mlx5e_tls_stats_desc, MLX5E_TLS_STATS_NUM,
167 ptls->stats.arg);
168
169 return (0);
170 }
171
172 void
mlx5e_tls_cleanup(struct mlx5e_priv * priv)173 mlx5e_tls_cleanup(struct mlx5e_priv *priv)
174 {
175 struct mlx5e_tls *ptls = &priv->tls;
176 uint32_t x;
177
178 if (ptls->init == 0)
179 return;
180
181 ptls->init = 0;
182 flush_workqueue(ptls->wq);
183 sysctl_ctx_free(&ptls->ctx);
184 uma_zdestroy(ptls->zone);
185 destroy_workqueue(ptls->wq);
186
187 /* check if all resources are freed */
188 MPASS(priv->tls.num_resources == 0);
189
190 for (x = 0; x != MLX5E_TLS_STATS_NUM; x++)
191 counter_u64_free(ptls->stats.arg[x]);
192 }
193
194 static void
mlx5e_tls_work(struct work_struct * work)195 mlx5e_tls_work(struct work_struct *work)
196 {
197 struct mlx5e_tls_tag *ptag;
198 struct mlx5e_priv *priv;
199 int err;
200
201 ptag = container_of(work, struct mlx5e_tls_tag, work);
202 priv = container_of(ptag->tls, struct mlx5e_priv, tls);
203
204 switch (ptag->state) {
205 case MLX5E_TLS_ST_INIT:
206 /* try to open TIS, if not present */
207 if (ptag->tisn == 0) {
208 err = mlx5_tls_open_tis(priv->mdev, 0, priv->tdn,
209 priv->pdn, &ptag->tisn);
210 if (err) {
211 MLX5E_TLS_STAT_INC(ptag, tx_error, 1);
212 break;
213 }
214 }
215 MLX5_SET(sw_tls_cntx, ptag->crypto_params, progress.pd, ptag->tisn);
216
217 /* try to allocate a DEK context ID */
218 err = mlx5_encryption_key_create(priv->mdev, priv->pdn,
219 MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, key.key_data),
220 MLX5_GET(sw_tls_cntx, ptag->crypto_params, key.key_len),
221 &ptag->dek_index);
222 if (err) {
223 MLX5E_TLS_STAT_INC(ptag, tx_error, 1);
224 break;
225 }
226
227 MLX5_SET(sw_tls_cntx, ptag->crypto_params, param.dek_index, ptag->dek_index);
228
229 ptag->dek_index_ok = 1;
230
231 MLX5E_TLS_TAG_LOCK(ptag);
232 if (ptag->state == MLX5E_TLS_ST_INIT)
233 ptag->state = MLX5E_TLS_ST_SETUP;
234 MLX5E_TLS_TAG_UNLOCK(ptag);
235 break;
236
237 case MLX5E_TLS_ST_RELEASE:
238 /* try to destroy DEK context by ID */
239 if (ptag->dek_index_ok)
240 err = mlx5_encryption_key_destroy(priv->mdev, ptag->dek_index);
241
242 /* free tag */
243 mlx5e_tls_tag_zfree(ptag);
244 break;
245
246 default:
247 break;
248 }
249 }
250
251 static int
mlx5e_tls_set_params(void * ctx,const struct tls_session_params * en)252 mlx5e_tls_set_params(void *ctx, const struct tls_session_params *en)
253 {
254
255 MLX5_SET(sw_tls_cntx, ctx, param.const_2, 2);
256 if (en->tls_vminor == TLS_MINOR_VER_TWO)
257 MLX5_SET(sw_tls_cntx, ctx, param.tls_version, 2); /* v1.2 */
258 else
259 MLX5_SET(sw_tls_cntx, ctx, param.tls_version, 3); /* v1.3 */
260 MLX5_SET(sw_tls_cntx, ctx, param.const_1, 1);
261 MLX5_SET(sw_tls_cntx, ctx, param.encryption_standard, 1); /* TLS */
262
263 /* copy the initial vector in place */
264 switch (en->iv_len) {
265 case MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.gcm_iv):
266 case MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.gcm_iv) +
267 MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.implicit_iv):
268 memcpy(MLX5_ADDR_OF(sw_tls_cntx, ctx, param.gcm_iv),
269 en->iv, en->iv_len);
270 break;
271 default:
272 return (EINVAL);
273 }
274
275 if (en->cipher_key_len <= MLX5_FLD_SZ_BYTES(sw_tls_cntx, key.key_data)) {
276 memcpy(MLX5_ADDR_OF(sw_tls_cntx, ctx, key.key_data),
277 en->cipher_key, en->cipher_key_len);
278 MLX5_SET(sw_tls_cntx, ctx, key.key_len, en->cipher_key_len);
279 } else {
280 return (EINVAL);
281 }
282 return (0);
283 }
284
285 /* Verify zero default */
286 CTASSERT(MLX5E_TLS_ST_INIT == 0);
287
288 int
mlx5e_tls_snd_tag_alloc(struct ifnet * ifp,union if_snd_tag_alloc_params * params,struct m_snd_tag ** ppmt)289 mlx5e_tls_snd_tag_alloc(struct ifnet *ifp,
290 union if_snd_tag_alloc_params *params,
291 struct m_snd_tag **ppmt)
292 {
293 union if_snd_tag_alloc_params rl_params;
294 struct mlx5e_priv *priv;
295 struct mlx5e_tls_tag *ptag;
296 const struct tls_session_params *en;
297 int error;
298
299 priv = ifp->if_softc;
300
301 if (priv->gone != 0 || priv->tls.init == 0)
302 return (EOPNOTSUPP);
303
304 /* allocate new tag from zone, if any */
305 ptag = uma_zalloc(priv->tls.zone, M_NOWAIT);
306 if (ptag == NULL)
307 return (ENOMEM);
308
309 /* sanity check default values */
310 MPASS(ptag->dek_index == 0);
311 MPASS(ptag->dek_index_ok == 0);
312
313 /* setup TLS tag */
314 ptag->tls = &priv->tls;
315
316 /* check if there is no TIS context */
317 if (ptag->tisn == 0) {
318 uint32_t value;
319
320 value = atomic_fetchadd_32(&priv->tls.num_resources, 1U);
321
322 /* check resource limits */
323 if (value >= priv->tls.max_resources) {
324 error = ENOMEM;
325 goto failure;
326 }
327 }
328
329 en = ¶ms->tls.tls->params;
330
331 /* only TLS v1.2 and v1.3 is currently supported */
332 if (en->tls_vmajor != TLS_MAJOR_VER_ONE ||
333 (en->tls_vminor != TLS_MINOR_VER_TWO
334 #ifdef TLS_MINOR_VER_THREE
335 && en->tls_vminor != TLS_MINOR_VER_THREE
336 #endif
337 )) {
338 error = EPROTONOSUPPORT;
339 goto failure;
340 }
341
342 switch (en->cipher_algorithm) {
343 case CRYPTO_AES_NIST_GCM_16:
344 switch (en->cipher_key_len) {
345 case 128 / 8:
346 if (en->tls_vminor == TLS_MINOR_VER_TWO) {
347 if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_128) == 0) {
348 error = EPROTONOSUPPORT;
349 goto failure;
350 }
351 } else {
352 if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_128) == 0) {
353 error = EPROTONOSUPPORT;
354 goto failure;
355 }
356 }
357 error = mlx5e_tls_set_params(ptag->crypto_params, en);
358 if (error)
359 goto failure;
360 break;
361
362 case 256 / 8:
363 if (en->tls_vminor == TLS_MINOR_VER_TWO) {
364 if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_256) == 0) {
365 error = EPROTONOSUPPORT;
366 goto failure;
367 }
368 } else {
369 if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_256) == 0) {
370 error = EPROTONOSUPPORT;
371 goto failure;
372 }
373 }
374 error = mlx5e_tls_set_params(ptag->crypto_params, en);
375 if (error)
376 goto failure;
377 break;
378
379 default:
380 error = EINVAL;
381 goto failure;
382 }
383 break;
384 default:
385 error = EPROTONOSUPPORT;
386 goto failure;
387 }
388
389 memset(&rl_params, 0, sizeof(rl_params));
390 rl_params.hdr = params->hdr;
391 switch (params->hdr.type) {
392 #ifdef RATELIMIT
393 case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
394 rl_params.hdr.type = IF_SND_TAG_TYPE_RATE_LIMIT;
395 rl_params.rate_limit.max_rate = params->tls_rate_limit.max_rate;
396 break;
397 #endif
398 case IF_SND_TAG_TYPE_TLS:
399 rl_params.hdr.type = IF_SND_TAG_TYPE_UNLIMITED;
400 break;
401 default:
402 error = EOPNOTSUPP;
403 goto failure;
404 }
405
406 error = m_snd_tag_alloc(ifp, &rl_params, &ptag->rl_tag);
407 if (error)
408 goto failure;
409
410 /* store pointer to mbuf tag */
411 MPASS(ptag->tag.refcount == 0);
412 m_snd_tag_init(&ptag->tag, ifp, params->hdr.type);
413 *ppmt = &ptag->tag;
414
415 /* reset state */
416 ptag->state = MLX5E_TLS_ST_INIT;
417
418 queue_work(priv->tls.wq, &ptag->work);
419 flush_work(&ptag->work);
420
421 return (0);
422
423 failure:
424 mlx5e_tls_tag_zfree(ptag);
425 return (error);
426 }
427
428 int
mlx5e_tls_snd_tag_modify(struct m_snd_tag * pmt,union if_snd_tag_modify_params * params)429 mlx5e_tls_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *params)
430 {
431 #ifdef RATELIMIT
432 union if_snd_tag_modify_params rl_params;
433 struct mlx5e_tls_tag *ptag =
434 container_of(pmt, struct mlx5e_tls_tag, tag);
435 int error;
436 #endif
437
438 switch (pmt->type) {
439 #ifdef RATELIMIT
440 case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
441 memset(&rl_params, 0, sizeof(rl_params));
442 rl_params.rate_limit.max_rate = params->tls_rate_limit.max_rate;
443 error = ptag->rl_tag->ifp->if_snd_tag_modify(ptag->rl_tag,
444 &rl_params);
445 return (error);
446 #endif
447 default:
448 return (EOPNOTSUPP);
449 }
450 }
451
452 int
mlx5e_tls_snd_tag_query(struct m_snd_tag * pmt,union if_snd_tag_query_params * params)453 mlx5e_tls_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params)
454 {
455 struct mlx5e_tls_tag *ptag =
456 container_of(pmt, struct mlx5e_tls_tag, tag);
457 int error;
458
459 switch (pmt->type) {
460 #ifdef RATELIMIT
461 case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
462 #endif
463 case IF_SND_TAG_TYPE_TLS:
464 error = ptag->rl_tag->ifp->if_snd_tag_query(ptag->rl_tag,
465 params);
466 break;
467 default:
468 error = EOPNOTSUPP;
469 break;
470 }
471 return (error);
472 }
473
474 void
mlx5e_tls_snd_tag_free(struct m_snd_tag * pmt)475 mlx5e_tls_snd_tag_free(struct m_snd_tag *pmt)
476 {
477 struct mlx5e_tls_tag *ptag =
478 container_of(pmt, struct mlx5e_tls_tag, tag);
479 struct mlx5e_priv *priv;
480
481 m_snd_tag_rele(ptag->rl_tag);
482
483 MLX5E_TLS_TAG_LOCK(ptag);
484 ptag->state = MLX5E_TLS_ST_RELEASE;
485 MLX5E_TLS_TAG_UNLOCK(ptag);
486
487 priv = ptag->tag.ifp->if_softc;
488 queue_work(priv->tls.wq, &ptag->work);
489 }
490
491 CTASSERT((MLX5_FLD_SZ_BYTES(sw_tls_cntx, param) % 16) == 0);
492
493 static void
mlx5e_tls_send_static_parameters(struct mlx5e_sq * sq,struct mlx5e_tls_tag * ptag)494 mlx5e_tls_send_static_parameters(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag)
495 {
496 const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_umr_wqe) +
497 MLX5_FLD_SZ_BYTES(sw_tls_cntx, param), MLX5_SEND_WQE_DS);
498 struct mlx5e_tx_umr_wqe *wqe;
499 u16 pi;
500
501 pi = sq->pc & sq->wq.sz_m1;
502 wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
503
504 memset(wqe, 0, sizeof(*wqe));
505
506 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) |
507 MLX5_OPCODE_UMR | (MLX5_OPCODE_MOD_UMR_TLS_TIS_STATIC_PARAMS << 24));
508 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
509 wqe->ctrl.imm = cpu_to_be32(ptag->tisn << 8);
510
511 if (mlx5e_do_send_cqe(sq))
512 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL;
513 else
514 wqe->ctrl.fm_ce_se = MLX5_FENCE_MODE_INITIATOR_SMALL;
515
516 /* fill out UMR control segment */
517 wqe->umr.flags = 0x80; /* inline data */
518 wqe->umr.bsf_octowords = cpu_to_be16(MLX5_FLD_SZ_BYTES(sw_tls_cntx, param) / 16);
519
520 /* copy in the static crypto parameters */
521 memcpy(wqe + 1, MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, param),
522 MLX5_FLD_SZ_BYTES(sw_tls_cntx, param));
523
524 /* copy data for doorbell */
525 memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32));
526
527 sq->mbuf[pi].mbuf = NULL;
528 sq->mbuf[pi].num_bytes = 0;
529 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
530 sq->mbuf[pi].mst = m_snd_tag_ref(&ptag->tag);
531
532 sq->pc += sq->mbuf[pi].num_wqebbs;
533 }
534
535 CTASSERT(MLX5_FLD_SZ_BYTES(sw_tls_cntx, progress) ==
536 sizeof(((struct mlx5e_tx_psv_wqe *)0)->psv));
537
538 static void
mlx5e_tls_send_progress_parameters(struct mlx5e_sq * sq,struct mlx5e_tls_tag * ptag)539 mlx5e_tls_send_progress_parameters(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag)
540 {
541 const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_psv_wqe),
542 MLX5_SEND_WQE_DS);
543 struct mlx5e_tx_psv_wqe *wqe;
544 u16 pi;
545
546 pi = sq->pc & sq->wq.sz_m1;
547 wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
548
549 memset(wqe, 0, sizeof(*wqe));
550
551 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) |
552 MLX5_OPCODE_SET_PSV | (MLX5_OPCODE_MOD_PSV_TLS_TIS_PROGRESS_PARAMS << 24));
553 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
554
555 if (mlx5e_do_send_cqe(sq))
556 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
557
558 /* copy in the PSV control segment */
559 memcpy(&wqe->psv, MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, progress),
560 sizeof(wqe->psv));
561
562 /* copy data for doorbell */
563 memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32));
564
565 sq->mbuf[pi].mbuf = NULL;
566 sq->mbuf[pi].num_bytes = 0;
567 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
568 sq->mbuf[pi].mst = m_snd_tag_ref(&ptag->tag);
569
570 sq->pc += sq->mbuf[pi].num_wqebbs;
571 }
572
573 static void
mlx5e_tls_send_nop(struct mlx5e_sq * sq,struct mlx5e_tls_tag * ptag)574 mlx5e_tls_send_nop(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag)
575 {
576 const u32 ds_cnt = MLX5_SEND_WQEBB_NUM_DS;
577 struct mlx5e_tx_wqe *wqe;
578 u16 pi;
579
580 pi = sq->pc & sq->wq.sz_m1;
581 wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
582
583 memset(&wqe->ctrl, 0, sizeof(wqe->ctrl));
584
585 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_NOP);
586 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
587 if (mlx5e_do_send_cqe(sq))
588 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL;
589 else
590 wqe->ctrl.fm_ce_se = MLX5_FENCE_MODE_INITIATOR_SMALL;
591
592 /* Copy data for doorbell */
593 memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32));
594
595 sq->mbuf[pi].mbuf = NULL;
596 sq->mbuf[pi].num_bytes = 0;
597 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
598 sq->mbuf[pi].mst = m_snd_tag_ref(&ptag->tag);
599
600 sq->pc += sq->mbuf[pi].num_wqebbs;
601 }
602
603 #define SBTLS_MBUF_NO_DATA ((struct mbuf *)1)
604
605 static struct mbuf *
sbtls_recover_record(struct mbuf * mb,int wait,uint32_t tcp_old,uint32_t * ptcp_seq,bool * pis_start)606 sbtls_recover_record(struct mbuf *mb, int wait, uint32_t tcp_old, uint32_t *ptcp_seq, bool *pis_start)
607 {
608 struct mbuf *mr, *top;
609 uint32_t offset;
610 uint32_t delta;
611
612 /* check format of incoming mbuf */
613 if (mb->m_next == NULL ||
614 (mb->m_next->m_flags & (M_EXTPG | M_EXT)) != (M_EXTPG | M_EXT)) {
615 top = NULL;
616 goto done;
617 }
618
619 /* get unmapped data offset */
620 offset = mtod(mb->m_next, uintptr_t);
621
622 /* check if we don't need to re-transmit anything */
623 if (offset == 0) {
624 top = SBTLS_MBUF_NO_DATA;
625 *pis_start = true;
626 goto done;
627 }
628
629 /* try to get a new packet header */
630 top = m_gethdr(wait, MT_DATA);
631 if (top == NULL)
632 goto done;
633
634 mr = m_get(wait, MT_DATA);
635 if (mr == NULL) {
636 m_free(top);
637 top = NULL;
638 goto done;
639 }
640
641 top->m_next = mr;
642
643 mb_dupcl(mr, mb->m_next);
644
645 /* the beginning of the TLS record */
646 mr->m_data = NULL;
647
648 /* setup packet header length */
649 top->m_pkthdr.len = mr->m_len = offset;
650 top->m_len = 0;
651
652 /* check for partial re-transmit */
653 delta = *ptcp_seq - tcp_old;
654
655 if (delta < offset) {
656 m_adj(top, offset - delta);
657 offset = delta;
658
659 /* continue where we left off */
660 *pis_start = false;
661 } else {
662 *pis_start = true;
663 }
664
665 /*
666 * Rewind the TCP sequence number by the amount of data
667 * retransmitted:
668 */
669 *ptcp_seq -= offset;
670 done:
671 return (top);
672 }
673
674 static int
mlx5e_sq_tls_populate(struct mbuf * mb,uint64_t * pseq)675 mlx5e_sq_tls_populate(struct mbuf *mb, uint64_t *pseq)
676 {
677
678 for (; mb != NULL; mb = mb->m_next) {
679 if (!(mb->m_flags & M_EXTPG))
680 continue;
681 *pseq = mb->m_epg_seqno;
682 return (1);
683 }
684 return (0);
685 }
686
687 int
mlx5e_sq_tls_xmit(struct mlx5e_sq * sq,struct mlx5e_xmit_args * parg,struct mbuf ** ppmb)688 mlx5e_sq_tls_xmit(struct mlx5e_sq *sq, struct mlx5e_xmit_args *parg, struct mbuf **ppmb)
689 {
690 struct mlx5e_tls_tag *ptls_tag;
691 struct m_snd_tag *ptag;
692 const struct tcphdr *th;
693 struct mbuf *mb = *ppmb;
694 u64 rcd_sn;
695 u32 header_size;
696 u32 mb_seq;
697
698 if ((mb->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0)
699 return (MLX5E_TLS_CONTINUE);
700
701 ptag = mb->m_pkthdr.snd_tag;
702
703 if (
704 #ifdef RATELIMIT
705 ptag->type != IF_SND_TAG_TYPE_TLS_RATE_LIMIT &&
706 #endif
707 ptag->type != IF_SND_TAG_TYPE_TLS)
708 return (MLX5E_TLS_CONTINUE);
709
710 ptls_tag = container_of(ptag, struct mlx5e_tls_tag, tag);
711
712 header_size = mlx5e_get_full_header_size(mb, &th);
713 if (unlikely(header_size == 0 || th == NULL))
714 return (MLX5E_TLS_FAILURE);
715
716 /*
717 * Send non-TLS TCP packets AS-IS:
718 */
719 if (header_size == mb->m_pkthdr.len ||
720 mlx5e_sq_tls_populate(mb, &rcd_sn) == 0) {
721 parg->tisn = 0;
722 parg->ihs = header_size;
723 return (MLX5E_TLS_CONTINUE);
724 }
725
726 mb_seq = ntohl(th->th_seq);
727
728 MLX5E_TLS_TAG_LOCK(ptls_tag);
729 switch (ptls_tag->state) {
730 case MLX5E_TLS_ST_INIT:
731 MLX5E_TLS_TAG_UNLOCK(ptls_tag);
732 return (MLX5E_TLS_FAILURE);
733 case MLX5E_TLS_ST_SETUP:
734 ptls_tag->state = MLX5E_TLS_ST_TXRDY;
735 ptls_tag->expected_seq = ~mb_seq; /* force setup */
736 default:
737 MLX5E_TLS_TAG_UNLOCK(ptls_tag);
738 break;
739 }
740
741 if (unlikely(ptls_tag->expected_seq != mb_seq)) {
742 bool is_start;
743 struct mbuf *r_mb;
744 uint32_t tcp_seq = mb_seq;
745
746 r_mb = sbtls_recover_record(mb, M_NOWAIT, ptls_tag->expected_seq, &tcp_seq, &is_start);
747 if (r_mb == NULL) {
748 MLX5E_TLS_STAT_INC(ptls_tag, tx_error, 1);
749 return (MLX5E_TLS_FAILURE);
750 }
751
752 MLX5E_TLS_STAT_INC(ptls_tag, tx_packets_ooo, 1);
753
754 /* check if this is the first fragment of a TLS record */
755 if (is_start) {
756 /* setup TLS static parameters */
757 MLX5_SET64(sw_tls_cntx, ptls_tag->crypto_params,
758 param.initial_record_number, rcd_sn);
759
760 /*
761 * NOTE: The sendqueue should have enough room to
762 * carry both the static and the progress parameters
763 * when we get here!
764 */
765 mlx5e_tls_send_static_parameters(sq, ptls_tag);
766 mlx5e_tls_send_progress_parameters(sq, ptls_tag);
767
768 if (r_mb == SBTLS_MBUF_NO_DATA) {
769 mlx5e_tls_send_nop(sq, ptls_tag);
770 ptls_tag->expected_seq = mb_seq;
771 return (MLX5E_TLS_LOOP);
772 }
773 }
774
775 MLX5E_TLS_STAT_INC(ptls_tag, tx_bytes_ooo, r_mb->m_pkthdr.len);
776
777 /* setup transmit arguments */
778 parg->tisn = ptls_tag->tisn;
779 parg->mst = &ptls_tag->tag;
780
781 /* try to send DUMP data */
782 if (mlx5e_sq_dump_xmit(sq, parg, &r_mb) != 0) {
783 m_freem(r_mb);
784 ptls_tag->expected_seq = tcp_seq;
785 return (MLX5E_TLS_FAILURE);
786 } else {
787 ptls_tag->expected_seq = mb_seq;
788 return (MLX5E_TLS_LOOP);
789 }
790 } else {
791 MLX5E_TLS_STAT_INC(ptls_tag, tx_packets, 1);
792 MLX5E_TLS_STAT_INC(ptls_tag, tx_bytes, mb->m_pkthdr.len);
793 }
794 ptls_tag->expected_seq += mb->m_pkthdr.len - header_size;
795
796 parg->tisn = ptls_tag->tisn;
797 parg->ihs = header_size;
798 parg->mst = &ptls_tag->tag;
799 return (MLX5E_TLS_CONTINUE);
800 }
801
802 #else
803
804 int
mlx5e_tls_init(struct mlx5e_priv * priv)805 mlx5e_tls_init(struct mlx5e_priv *priv)
806 {
807
808 return (0);
809 }
810
811 void
mlx5e_tls_cleanup(struct mlx5e_priv * priv)812 mlx5e_tls_cleanup(struct mlx5e_priv *priv)
813 {
814 /* NOP */
815 }
816
817 #endif /* KERN_TLS */
818