1 /*-
2 * Copyright (c) 2010-2011 Juniper Networks, Inc.
3 * All rights reserved.
4 *
5 * This software was developed by Robert N. M. Watson under contract
6 * to Juniper Networks, Inc.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30
31 #include "opt_inet6.h"
32 #include "opt_pcbgroup.h"
33
34 #ifndef PCBGROUP
35 #error "options RSS depends on options PCBGROUP"
36 #endif
37
38 #include <sys/param.h>
39 #include <sys/mbuf.h>
40 #include <sys/socket.h>
41 #include <sys/priv.h>
42 #include <sys/kernel.h>
43 #include <sys/smp.h>
44 #include <sys/sysctl.h>
45 #include <sys/sbuf.h>
46
47 #include <net/if.h>
48 #include <net/if_var.h>
49 #include <net/netisr.h>
50 #include <net/rss_config.h>
51 #include <net/toeplitz.h>
52
53 /*-
54 * Operating system parts of receiver-side scaling (RSS), which allows
55 * network cards to direct flows to particular receive queues based on hashes
56 * of header tuples. This implementation aligns RSS buckets with connection
57 * groups at the TCP/IP layer, so each bucket is associated with exactly one
58 * group. As a result, the group lookup structures (and lock) should have an
59 * effective affinity with exactly one CPU.
60 *
61 * Network device drivers needing to configure RSS will query this framework
62 * for parameters, such as the current RSS key, hashing policies, number of
63 * bits, and indirection table mapping hashes to buckets and CPUs. They may
64 * provide their own supplementary information, such as queue<->CPU bindings.
65 * It is the responsibility of the network device driver to inject packets
66 * into the stack on as close to the right CPU as possible, if playing by RSS
67 * rules.
68 *
69 * TODO:
70 *
71 * - Synchronization for rss_key and other future-configurable parameters.
72 * - Event handler drivers can register to pick up RSS configuration changes.
73 * - Should we allow rss_basecpu to be configured?
74 * - Randomize key on boot.
75 * - IPv6 support.
76 * - Statistics on how often there's a misalignment between hardware
77 * placement and pcbgroup expectations.
78 */
79
80 SYSCTL_DECL(_net_inet);
81 SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
82 "Receive-side steering");
83
84 /*
85 * Toeplitz is the only required hash function in the RSS spec, so use it by
86 * default.
87 */
88 static u_int rss_hashalgo = RSS_HASH_TOEPLITZ;
89 SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0,
90 "RSS hash algorithm");
91
92 /*
93 * Size of the indirection table; at most 128 entries per the RSS spec. We
94 * size it to at least 2 times the number of CPUs by default to allow useful
95 * rebalancing. If not set explicitly with a loader tunable, we tune based
96 * on the number of CPUs present.
97 *
98 * XXXRW: buckets might be better to use for the tunable than bits.
99 */
100 static u_int rss_bits;
101 SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0,
102 "RSS bits");
103
104 static u_int rss_mask;
105 SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0,
106 "RSS mask");
107
108 static const u_int rss_maxbits = RSS_MAXBITS;
109 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD,
110 __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits");
111
112 /*
113 * RSS's own count of the number of CPUs it could be using for processing.
114 * Bounded to 64 by RSS constants.
115 */
116 static u_int rss_ncpus;
117 SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0,
118 "Number of CPUs available to RSS");
119
120 #define RSS_MAXCPUS (1 << (RSS_MAXBITS - 1))
121 static const u_int rss_maxcpus = RSS_MAXCPUS;
122 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD,
123 __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used");
124
125 /*
126 * Variable exists just for reporting rss_bits in a user-friendly way.
127 */
128 static u_int rss_buckets;
129 SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0,
130 "RSS buckets");
131
132 /*
133 * Base CPU number; devices will add this to all CPU numbers returned by the
134 * RSS indirection table. Currently unmodifable in FreeBSD.
135 */
136 static const u_int rss_basecpu;
137 SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD,
138 __DECONST(int *, &rss_basecpu), 0, "RSS base CPU");
139
140 /*
141 * Print verbose debugging messages.
142 * 0 - disable
143 * non-zero - enable
144 */
145 int rss_debug = 0;
146 SYSCTL_INT(_net_inet_rss, OID_AUTO, debug, CTLFLAG_RWTUN, &rss_debug, 0,
147 "RSS debug level");
148
149 /*
150 * RSS secret key, intended to prevent attacks on load-balancing. Its
151 * effectiveness may be limited by algorithm choice and available entropy
152 * during the boot.
153 *
154 * XXXRW: And that we don't randomize it yet!
155 *
156 * This is the default Microsoft RSS specification key which is also
157 * the Chelsio T5 firmware default key.
158 */
159 static uint8_t rss_key[RSS_KEYSIZE] = {
160 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
161 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
162 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
163 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
164 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa,
165 };
166
167 /*
168 * RSS hash->CPU table, which maps hashed packet headers to particular CPUs.
169 * Drivers may supplement this table with a separate CPU<->queue table when
170 * programming devices.
171 */
172 struct rss_table_entry {
173 uint8_t rte_cpu; /* CPU affinity of bucket. */
174 };
175 static struct rss_table_entry rss_table[RSS_TABLE_MAXLEN];
176
177 static void
rss_init(__unused void * arg)178 rss_init(__unused void *arg)
179 {
180 u_int i;
181 u_int cpuid;
182
183 /*
184 * Validate tunables, coerce to sensible values.
185 */
186 switch (rss_hashalgo) {
187 case RSS_HASH_TOEPLITZ:
188 case RSS_HASH_NAIVE:
189 break;
190
191 default:
192 RSS_DEBUG("invalid RSS hashalgo %u, coercing to %u\n",
193 rss_hashalgo, RSS_HASH_TOEPLITZ);
194 rss_hashalgo = RSS_HASH_TOEPLITZ;
195 }
196
197 /*
198 * Count available CPUs.
199 *
200 * XXXRW: Note incorrect assumptions regarding contiguity of this set
201 * elsewhere.
202 */
203 rss_ncpus = 0;
204 for (i = 0; i <= mp_maxid; i++) {
205 if (CPU_ABSENT(i))
206 continue;
207 rss_ncpus++;
208 }
209 if (rss_ncpus > RSS_MAXCPUS)
210 rss_ncpus = RSS_MAXCPUS;
211
212 /*
213 * Tune RSS table entries to be no less than 2x the number of CPUs
214 * -- unless we're running uniprocessor, in which case there's not
215 * much point in having buckets to rearrange for load-balancing!
216 */
217 if (rss_ncpus > 1) {
218 if (rss_bits == 0)
219 rss_bits = fls(rss_ncpus - 1) + 1;
220
221 /*
222 * Microsoft limits RSS table entries to 128, so apply that
223 * limit to both auto-detected CPU counts and user-configured
224 * ones.
225 */
226 if (rss_bits == 0 || rss_bits > RSS_MAXBITS) {
227 RSS_DEBUG("RSS bits %u not valid, coercing to %u\n",
228 rss_bits, RSS_MAXBITS);
229 rss_bits = RSS_MAXBITS;
230 }
231
232 /*
233 * Figure out how many buckets to use; warn if less than the
234 * number of configured CPUs, although this is not a fatal
235 * problem.
236 */
237 rss_buckets = (1 << rss_bits);
238 if (rss_buckets < rss_ncpus)
239 RSS_DEBUG("WARNING: rss_buckets (%u) less than "
240 "rss_ncpus (%u)\n", rss_buckets, rss_ncpus);
241 rss_mask = rss_buckets - 1;
242 } else {
243 rss_bits = 0;
244 rss_buckets = 1;
245 rss_mask = 0;
246 }
247
248 /*
249 * Set up initial CPU assignments: round-robin by default.
250 */
251 cpuid = CPU_FIRST();
252 for (i = 0; i < rss_buckets; i++) {
253 rss_table[i].rte_cpu = cpuid;
254 cpuid = CPU_NEXT(cpuid);
255 }
256
257 /*
258 * Randomize rrs_key.
259 *
260 * XXXRW: Not yet. If nothing else, will require an rss_isbadkey()
261 * loop to check for "bad" RSS keys.
262 */
263 }
264 SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL);
265
266 static uint32_t
rss_naive_hash(u_int keylen,const uint8_t * key,u_int datalen,const uint8_t * data)267 rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen,
268 const uint8_t *data)
269 {
270 uint32_t v;
271 u_int i;
272
273 v = 0;
274 for (i = 0; i < keylen; i++)
275 v += key[i];
276 for (i = 0; i < datalen; i++)
277 v += data[i];
278 return (v);
279 }
280
281 uint32_t
rss_hash(u_int datalen,const uint8_t * data)282 rss_hash(u_int datalen, const uint8_t *data)
283 {
284
285 switch (rss_hashalgo) {
286 case RSS_HASH_TOEPLITZ:
287 return (toeplitz_hash(sizeof(rss_key), rss_key, datalen,
288 data));
289
290 case RSS_HASH_NAIVE:
291 return (rss_naive_hash(sizeof(rss_key), rss_key, datalen,
292 data));
293
294 default:
295 panic("%s: unsupported/unknown hashalgo %d", __func__,
296 rss_hashalgo);
297 }
298 }
299
300 /*
301 * Query the number of RSS bits in use.
302 */
303 u_int
rss_getbits(void)304 rss_getbits(void)
305 {
306
307 return (rss_bits);
308 }
309
310 /*
311 * Query the RSS bucket associated with an RSS hash.
312 */
313 u_int
rss_getbucket(u_int hash)314 rss_getbucket(u_int hash)
315 {
316
317 return (hash & rss_mask);
318 }
319
320 /*
321 * Query the RSS layer bucket associated with the given
322 * entry in the RSS hash space.
323 *
324 * The RSS indirection table is 0 .. rss_buckets-1,
325 * covering the low 'rss_bits' of the total 128 slot
326 * RSS indirection table. So just mask off rss_bits and
327 * return that.
328 *
329 * NIC drivers can then iterate over the 128 slot RSS
330 * indirection table and fetch which RSS bucket to
331 * map it to. This will typically be a CPU queue
332 */
333 u_int
rss_get_indirection_to_bucket(u_int index)334 rss_get_indirection_to_bucket(u_int index)
335 {
336
337 return (index & rss_mask);
338 }
339
340 /*
341 * Query the RSS CPU associated with an RSS bucket.
342 */
343 u_int
rss_getcpu(u_int bucket)344 rss_getcpu(u_int bucket)
345 {
346
347 return (rss_table[bucket].rte_cpu);
348 }
349
350 /*
351 * netisr CPU affinity lookup given just the hash and hashtype.
352 */
353 u_int
rss_hash2cpuid(uint32_t hash_val,uint32_t hash_type)354 rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type)
355 {
356
357 switch (hash_type) {
358 case M_HASHTYPE_RSS_IPV4:
359 case M_HASHTYPE_RSS_TCP_IPV4:
360 case M_HASHTYPE_RSS_UDP_IPV4:
361 case M_HASHTYPE_RSS_IPV6:
362 case M_HASHTYPE_RSS_TCP_IPV6:
363 case M_HASHTYPE_RSS_UDP_IPV6:
364 return (rss_getcpu(rss_getbucket(hash_val)));
365 default:
366 return (NETISR_CPUID_NONE);
367 }
368 }
369
370 /*
371 * Query the RSS bucket associated with the given hash value and
372 * type.
373 */
374 int
rss_hash2bucket(uint32_t hash_val,uint32_t hash_type,uint32_t * bucket_id)375 rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id)
376 {
377
378 switch (hash_type) {
379 case M_HASHTYPE_RSS_IPV4:
380 case M_HASHTYPE_RSS_TCP_IPV4:
381 case M_HASHTYPE_RSS_UDP_IPV4:
382 case M_HASHTYPE_RSS_IPV6:
383 case M_HASHTYPE_RSS_TCP_IPV6:
384 case M_HASHTYPE_RSS_UDP_IPV6:
385 *bucket_id = rss_getbucket(hash_val);
386 return (0);
387 default:
388 return (-1);
389 }
390 }
391
392 /*
393 * netisr CPU affinity lookup routine for use by protocols.
394 */
395 struct mbuf *
rss_m2cpuid(struct mbuf * m,uintptr_t source,u_int * cpuid)396 rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid)
397 {
398
399 M_ASSERTPKTHDR(m);
400 *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m));
401 return (m);
402 }
403
404 int
rss_m2bucket(struct mbuf * m,uint32_t * bucket_id)405 rss_m2bucket(struct mbuf *m, uint32_t *bucket_id)
406 {
407
408 M_ASSERTPKTHDR(m);
409
410 return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m),
411 bucket_id));
412 }
413
414 /*
415 * Query the RSS hash algorithm.
416 */
417 u_int
rss_gethashalgo(void)418 rss_gethashalgo(void)
419 {
420
421 return (rss_hashalgo);
422 }
423
424 /*
425 * Query the current RSS key; likely to be used by device drivers when
426 * configuring hardware RSS. Caller must pass an array of size RSS_KEYSIZE.
427 *
428 * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing?
429 */
430 void
rss_getkey(uint8_t * key)431 rss_getkey(uint8_t *key)
432 {
433
434 bcopy(rss_key, key, sizeof(rss_key));
435 }
436
437 /*
438 * Query the number of buckets; this may be used by both network device
439 * drivers, which will need to populate hardware shadows of the software
440 * indirection table, and the network stack itself (such as when deciding how
441 * many connection groups to allocate).
442 */
443 u_int
rss_getnumbuckets(void)444 rss_getnumbuckets(void)
445 {
446
447 return (rss_buckets);
448 }
449
450 /*
451 * Query the number of CPUs in use by RSS; may be useful to device drivers
452 * trying to figure out how to map a larger number of CPUs into a smaller
453 * number of receive queues.
454 */
455 u_int
rss_getnumcpus(void)456 rss_getnumcpus(void)
457 {
458
459 return (rss_ncpus);
460 }
461
462 /*
463 * Return the supported RSS hash configuration.
464 *
465 * NICs should query this to determine what to configure in their redirection
466 * matching table.
467 */
468 inline u_int
rss_gethashconfig(void)469 rss_gethashconfig(void)
470 {
471
472 /* Return 4-tuple for TCP; 2-tuple for others */
473 /*
474 * UDP may fragment more often than TCP and thus we'll end up with
475 * NICs returning 2-tuple fragments.
476 * udp_init() and udplite_init() both currently initialise things
477 * as 2-tuple.
478 * So for now disable UDP 4-tuple hashing until all of the other
479 * pieces are in place.
480 */
481 return (
482 RSS_HASHTYPE_RSS_IPV4
483 | RSS_HASHTYPE_RSS_TCP_IPV4
484 | RSS_HASHTYPE_RSS_IPV6
485 | RSS_HASHTYPE_RSS_TCP_IPV6
486 | RSS_HASHTYPE_RSS_IPV6_EX
487 | RSS_HASHTYPE_RSS_TCP_IPV6_EX
488 #if 0
489 | RSS_HASHTYPE_RSS_UDP_IPV4
490 | RSS_HASHTYPE_RSS_UDP_IPV6
491 | RSS_HASHTYPE_RSS_UDP_IPV6_EX
492 #endif
493 );
494 }
495
496 /*
497 * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want
498 * it appearing in debugging output unnecessarily.
499 */
500 static int
sysctl_rss_key(SYSCTL_HANDLER_ARGS)501 sysctl_rss_key(SYSCTL_HANDLER_ARGS)
502 {
503 uint8_t temp_rss_key[RSS_KEYSIZE];
504 int error;
505
506 error = priv_check(req->td, PRIV_NETINET_HASHKEY);
507 if (error)
508 return (error);
509
510 bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key));
511 error = sysctl_handle_opaque(oidp, temp_rss_key,
512 sizeof(temp_rss_key), req);
513 if (error)
514 return (error);
515 if (req->newptr != NULL) {
516 /* XXXRW: Not yet. */
517 return (EINVAL);
518 }
519 return (0);
520 }
521 SYSCTL_PROC(_net_inet_rss, OID_AUTO, key,
522 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key,
523 "", "RSS keying material");
524
525 static int
sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS)526 sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS)
527 {
528 struct sbuf *sb;
529 int error;
530 int i;
531
532 error = 0;
533 error = sysctl_wire_old_buffer(req, 0);
534 if (error != 0)
535 return (error);
536 sb = sbuf_new_for_sysctl(NULL, NULL, 512, req);
537 if (sb == NULL)
538 return (ENOMEM);
539 for (i = 0; i < rss_buckets; i++) {
540 sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ",
541 i,
542 rss_getcpu(i));
543 }
544 error = sbuf_finish(sb);
545 sbuf_delete(sb);
546
547 return (error);
548 }
549 SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping,
550 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
551 sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping");
552