1 /*
2 * Copyright (c) 2008 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
34 * $DragonFly: src/sys/net/altq/altq_fairq.c,v 1.1 2008/04/06 18:58:15 dillon Exp $
35 */
36 /*
37 * Matt: I gutted altq_priq.c and used it as a skeleton on which to build
38 * fairq. The fairq algorithm is completely different then priq, of course,
39 * but because I used priq's skeleton I believe I should include priq's
40 * copyright.
41 *
42 * Copyright (C) 2000-2003
43 * Sony Computer Science Laboratories Inc. All rights reserved.
44 *
45 * Redistribution and use in source and binary forms, with or without
46 * modification, are permitted provided that the following conditions
47 * are met:
48 * 1. Redistributions of source code must retain the above copyright
49 * notice, this list of conditions and the following disclaimer.
50 * 2. Redistributions in binary form must reproduce the above copyright
51 * notice, this list of conditions and the following disclaimer in the
52 * documentation and/or other materials provided with the distribution.
53 *
54 * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND
55 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
56 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
57 * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE
58 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
59 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
60 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
61 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
62 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
63 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
64 * SUCH DAMAGE.
65 */
66
67 /*
68 * FAIRQ - take traffic classified by keep state (hashed into
69 * mbuf->m_pkthdr.altq_state_hash) and bucketize it. Fairly extract
70 * the first packet from each bucket in a round-robin fashion.
71 *
72 * TODO - better overall qlimit support (right now it is per-bucket).
73 * - NOTE: red etc is per bucket, not overall.
74 * - better service curve support.
75 *
76 * EXAMPLE:
77 *
78 * altq on em0 fairq bandwidth 650Kb queue { std, bulk }
79 * queue std priority 3 bandwidth 400Kb \
80 * fairq (buckets 64, default, hogs 1Kb) qlimit 50
81 * queue bulk priority 2 bandwidth 100Kb \
82 * fairq (buckets 64, hogs 1Kb) qlimit 50
83 *
84 * pass out on em0 from any to any keep state queue std
85 * pass out on em0 inet proto tcp ..... port ... keep state queue bulk
86 */
87 #include "opt_altq.h"
88 #include "opt_inet.h"
89 #include "opt_inet6.h"
90
91 #ifdef ALTQ_FAIRQ /* fairq is enabled in the kernel conf */
92
93 #include <sys/param.h>
94 #include <sys/malloc.h>
95 #include <sys/mbuf.h>
96 #include <sys/socket.h>
97 #include <sys/sockio.h>
98 #include <sys/systm.h>
99 #include <sys/proc.h>
100 #include <sys/errno.h>
101 #include <sys/kernel.h>
102 #include <sys/queue.h>
103
104 #include <net/if.h>
105 #include <net/if_var.h>
106 #include <netinet/in.h>
107
108 #include <netpfil/pf/pf.h>
109 #include <netpfil/pf/pf_altq.h>
110 #include <netpfil/pf/pf_mtag.h>
111 #include <net/altq/altq.h>
112 #include <net/altq/altq_fairq.h>
113
114 /*
115 * function prototypes
116 */
117 static int fairq_clear_interface(struct fairq_if *);
118 static int fairq_request(struct ifaltq *, int, void *);
119 static void fairq_purge(struct fairq_if *);
120 static struct fairq_class *fairq_class_create(struct fairq_if *, int, int, u_int, struct fairq_opts *, int);
121 static int fairq_class_destroy(struct fairq_class *);
122 static int fairq_enqueue(struct ifaltq *, struct mbuf *, struct altq_pktattr *);
123 static struct mbuf *fairq_dequeue(struct ifaltq *, int);
124
125 static int fairq_addq(struct fairq_class *, struct mbuf *, u_int32_t);
126 static struct mbuf *fairq_getq(struct fairq_class *, uint64_t);
127 static struct mbuf *fairq_pollq(struct fairq_class *, uint64_t, int *);
128 static fairq_bucket_t *fairq_selectq(struct fairq_class *, int);
129 static void fairq_purgeq(struct fairq_class *);
130
131 static void get_class_stats(struct fairq_classstats *, struct fairq_class *);
132 static struct fairq_class *clh_to_clp(struct fairq_if *, uint32_t);
133
134 int
fairq_pfattach(struct pf_altq * a)135 fairq_pfattach(struct pf_altq *a)
136 {
137 struct ifnet *ifp;
138 int error;
139
140 if ((ifp = ifunit(a->ifname)) == NULL || a->altq_disc == NULL)
141 return (EINVAL);
142
143 error = altq_attach(&ifp->if_snd, ALTQT_FAIRQ, a->altq_disc,
144 fairq_enqueue, fairq_dequeue, fairq_request, NULL, NULL);
145
146 return (error);
147 }
148
149 int
fairq_add_altq(struct ifnet * ifp,struct pf_altq * a)150 fairq_add_altq(struct ifnet *ifp, struct pf_altq *a)
151 {
152 struct fairq_if *pif;
153
154 if (ifp == NULL)
155 return (EINVAL);
156 if (!ALTQ_IS_READY(&ifp->if_snd))
157 return (ENODEV);
158
159 pif = malloc(sizeof(struct fairq_if),
160 M_DEVBUF, M_WAITOK | M_ZERO);
161 pif->pif_bandwidth = a->ifbandwidth;
162 pif->pif_maxpri = -1;
163 pif->pif_ifq = &ifp->if_snd;
164
165 /* keep the state in pf_altq */
166 a->altq_disc = pif;
167
168 return (0);
169 }
170
171 int
fairq_remove_altq(struct pf_altq * a)172 fairq_remove_altq(struct pf_altq *a)
173 {
174 struct fairq_if *pif;
175
176 if ((pif = a->altq_disc) == NULL)
177 return (EINVAL);
178 a->altq_disc = NULL;
179
180 fairq_clear_interface(pif);
181
182 free(pif, M_DEVBUF);
183 return (0);
184 }
185
186 int
fairq_add_queue(struct pf_altq * a)187 fairq_add_queue(struct pf_altq *a)
188 {
189 struct fairq_if *pif;
190 struct fairq_class *cl;
191
192 if ((pif = a->altq_disc) == NULL)
193 return (EINVAL);
194
195 /* check parameters */
196 if (a->priority >= FAIRQ_MAXPRI)
197 return (EINVAL);
198 if (a->qid == 0)
199 return (EINVAL);
200 if (pif->pif_classes[a->priority] != NULL)
201 return (EBUSY);
202 if (clh_to_clp(pif, a->qid) != NULL)
203 return (EBUSY);
204
205 cl = fairq_class_create(pif, a->priority, a->qlimit, a->bandwidth,
206 &a->pq_u.fairq_opts, a->qid);
207 if (cl == NULL)
208 return (ENOMEM);
209
210 return (0);
211 }
212
213 int
fairq_remove_queue(struct pf_altq * a)214 fairq_remove_queue(struct pf_altq *a)
215 {
216 struct fairq_if *pif;
217 struct fairq_class *cl;
218
219 if ((pif = a->altq_disc) == NULL)
220 return (EINVAL);
221
222 if ((cl = clh_to_clp(pif, a->qid)) == NULL)
223 return (EINVAL);
224
225 return (fairq_class_destroy(cl));
226 }
227
228 int
fairq_getqstats(struct pf_altq * a,void * ubuf,int * nbytes,int version)229 fairq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes, int version)
230 {
231 struct fairq_if *pif;
232 struct fairq_class *cl;
233 struct fairq_classstats stats;
234 int error = 0;
235
236 if ((pif = altq_lookup(a->ifname, ALTQT_FAIRQ)) == NULL)
237 return (EBADF);
238
239 if ((cl = clh_to_clp(pif, a->qid)) == NULL)
240 return (EINVAL);
241
242 if (*nbytes < sizeof(stats))
243 return (EINVAL);
244
245 get_class_stats(&stats, cl);
246
247 if ((error = copyout((caddr_t)&stats, ubuf, sizeof(stats))) != 0)
248 return (error);
249 *nbytes = sizeof(stats);
250 return (0);
251 }
252
253 /*
254 * bring the interface back to the initial state by discarding
255 * all the filters and classes.
256 */
257 static int
fairq_clear_interface(struct fairq_if * pif)258 fairq_clear_interface(struct fairq_if *pif)
259 {
260 struct fairq_class *cl;
261 int pri;
262
263 /* clear out the classes */
264 for (pri = 0; pri <= pif->pif_maxpri; pri++) {
265 if ((cl = pif->pif_classes[pri]) != NULL)
266 fairq_class_destroy(cl);
267 }
268
269 return (0);
270 }
271
272 static int
fairq_request(struct ifaltq * ifq,int req,void * arg)273 fairq_request(struct ifaltq *ifq, int req, void *arg)
274 {
275 struct fairq_if *pif = (struct fairq_if *)ifq->altq_disc;
276
277 IFQ_LOCK_ASSERT(ifq);
278
279 switch (req) {
280 case ALTRQ_PURGE:
281 fairq_purge(pif);
282 break;
283 }
284 return (0);
285 }
286
287 /* discard all the queued packets on the interface */
288 static void
fairq_purge(struct fairq_if * pif)289 fairq_purge(struct fairq_if *pif)
290 {
291 struct fairq_class *cl;
292 int pri;
293
294 for (pri = 0; pri <= pif->pif_maxpri; pri++) {
295 if ((cl = pif->pif_classes[pri]) != NULL && cl->cl_head)
296 fairq_purgeq(cl);
297 }
298 if (ALTQ_IS_ENABLED(pif->pif_ifq))
299 pif->pif_ifq->ifq_len = 0;
300 }
301
302 static struct fairq_class *
fairq_class_create(struct fairq_if * pif,int pri,int qlimit,u_int bandwidth,struct fairq_opts * opts,int qid)303 fairq_class_create(struct fairq_if *pif, int pri, int qlimit,
304 u_int bandwidth, struct fairq_opts *opts, int qid)
305 {
306 struct fairq_class *cl;
307 int flags = opts->flags;
308 u_int nbuckets = opts->nbuckets;
309 int i;
310
311 #ifndef ALTQ_RED
312 if (flags & FARF_RED) {
313 #ifdef ALTQ_DEBUG
314 printf("fairq_class_create: RED not configured for FAIRQ!\n");
315 #endif
316 return (NULL);
317 }
318 #endif
319 #ifndef ALTQ_CODEL
320 if (flags & FARF_CODEL) {
321 #ifdef ALTQ_DEBUG
322 printf("fairq_class_create: CODEL not configured for FAIRQ!\n");
323 #endif
324 return (NULL);
325 }
326 #endif
327 if (nbuckets == 0)
328 nbuckets = 256;
329 if (nbuckets > FAIRQ_MAX_BUCKETS)
330 nbuckets = FAIRQ_MAX_BUCKETS;
331 /* enforce power-of-2 size */
332 while ((nbuckets ^ (nbuckets - 1)) != ((nbuckets << 1) - 1))
333 ++nbuckets;
334
335 if ((cl = pif->pif_classes[pri]) != NULL) {
336 /* modify the class instead of creating a new one */
337 IFQ_LOCK(cl->cl_pif->pif_ifq);
338 if (cl->cl_head)
339 fairq_purgeq(cl);
340 IFQ_UNLOCK(cl->cl_pif->pif_ifq);
341 #ifdef ALTQ_RIO
342 if (cl->cl_qtype == Q_RIO)
343 rio_destroy((rio_t *)cl->cl_red);
344 #endif
345 #ifdef ALTQ_RED
346 if (cl->cl_qtype == Q_RED)
347 red_destroy(cl->cl_red);
348 #endif
349 #ifdef ALTQ_CODEL
350 if (cl->cl_qtype == Q_CODEL)
351 codel_destroy(cl->cl_codel);
352 #endif
353 } else {
354 cl = malloc(sizeof(struct fairq_class),
355 M_DEVBUF, M_WAITOK | M_ZERO);
356 cl->cl_nbuckets = nbuckets;
357 cl->cl_nbucket_mask = nbuckets - 1;
358
359 cl->cl_buckets = malloc(
360 sizeof(struct fairq_bucket) * cl->cl_nbuckets,
361 M_DEVBUF, M_WAITOK | M_ZERO);
362 cl->cl_head = NULL;
363 }
364
365 pif->pif_classes[pri] = cl;
366 if (flags & FARF_DEFAULTCLASS)
367 pif->pif_default = cl;
368 if (qlimit == 0)
369 qlimit = 50; /* use default */
370 cl->cl_qlimit = qlimit;
371 for (i = 0; i < cl->cl_nbuckets; ++i) {
372 qlimit(&cl->cl_buckets[i].queue) = qlimit;
373 }
374 cl->cl_bandwidth = bandwidth / 8;
375 cl->cl_qtype = Q_DROPTAIL;
376 cl->cl_flags = flags & FARF_USERFLAGS;
377 cl->cl_pri = pri;
378 if (pri > pif->pif_maxpri)
379 pif->pif_maxpri = pri;
380 cl->cl_pif = pif;
381 cl->cl_handle = qid;
382 cl->cl_hogs_m1 = opts->hogs_m1 / 8;
383 cl->cl_lssc_m1 = opts->lssc_m1 / 8; /* NOT YET USED */
384
385 #ifdef ALTQ_RED
386 if (flags & (FARF_RED|FARF_RIO)) {
387 int red_flags, red_pkttime;
388
389 red_flags = 0;
390 if (flags & FARF_ECN)
391 red_flags |= REDF_ECN;
392 #ifdef ALTQ_RIO
393 if (flags & FARF_CLEARDSCP)
394 red_flags |= RIOF_CLEARDSCP;
395 #endif
396 if (pif->pif_bandwidth < 8)
397 red_pkttime = 1000 * 1000 * 1000; /* 1 sec */
398 else
399 red_pkttime = (int64_t)pif->pif_ifq->altq_ifp->if_mtu
400 * 1000 * 1000 * 1000 / (pif->pif_bandwidth / 8);
401 #ifdef ALTQ_RIO
402 if (flags & FARF_RIO) {
403 cl->cl_red = (red_t *)rio_alloc(0, NULL,
404 red_flags, red_pkttime);
405 if (cl->cl_red != NULL)
406 cl->cl_qtype = Q_RIO;
407 } else
408 #endif
409 if (flags & FARF_RED) {
410 cl->cl_red = red_alloc(0, 0,
411 cl->cl_qlimit * 10/100,
412 cl->cl_qlimit * 30/100,
413 red_flags, red_pkttime);
414 if (cl->cl_red != NULL)
415 cl->cl_qtype = Q_RED;
416 }
417 }
418 #endif /* ALTQ_RED */
419 #ifdef ALTQ_CODEL
420 if (flags & FARF_CODEL) {
421 cl->cl_codel = codel_alloc(5, 100, 0);
422 if (cl->cl_codel != NULL)
423 cl->cl_qtype = Q_CODEL;
424 }
425 #endif
426
427 return (cl);
428 }
429
430 static int
fairq_class_destroy(struct fairq_class * cl)431 fairq_class_destroy(struct fairq_class *cl)
432 {
433 struct fairq_if *pif;
434 int pri;
435
436 IFQ_LOCK(cl->cl_pif->pif_ifq);
437
438 if (cl->cl_head)
439 fairq_purgeq(cl);
440
441 pif = cl->cl_pif;
442 pif->pif_classes[cl->cl_pri] = NULL;
443 if (pif->pif_poll_cache == cl)
444 pif->pif_poll_cache = NULL;
445 if (pif->pif_maxpri == cl->cl_pri) {
446 for (pri = cl->cl_pri; pri >= 0; pri--)
447 if (pif->pif_classes[pri] != NULL) {
448 pif->pif_maxpri = pri;
449 break;
450 }
451 if (pri < 0)
452 pif->pif_maxpri = -1;
453 }
454 IFQ_UNLOCK(cl->cl_pif->pif_ifq);
455
456 if (cl->cl_red != NULL) {
457 #ifdef ALTQ_RIO
458 if (cl->cl_qtype == Q_RIO)
459 rio_destroy((rio_t *)cl->cl_red);
460 #endif
461 #ifdef ALTQ_RED
462 if (cl->cl_qtype == Q_RED)
463 red_destroy(cl->cl_red);
464 #endif
465 #ifdef ALTQ_CODEL
466 if (cl->cl_qtype == Q_CODEL)
467 codel_destroy(cl->cl_codel);
468 #endif
469 }
470 free(cl->cl_buckets, M_DEVBUF);
471 free(cl, M_DEVBUF);
472
473 return (0);
474 }
475
476 /*
477 * fairq_enqueue is an enqueue function to be registered to
478 * (*altq_enqueue) in struct ifaltq.
479 */
480 static int
fairq_enqueue(struct ifaltq * ifq,struct mbuf * m,struct altq_pktattr * pktattr)481 fairq_enqueue(struct ifaltq *ifq, struct mbuf *m, struct altq_pktattr *pktattr)
482 {
483 struct fairq_if *pif = (struct fairq_if *)ifq->altq_disc;
484 struct fairq_class *cl = NULL; /* Make compiler happy */
485 struct pf_mtag *t;
486 u_int32_t qid_hash = 0;
487 int len;
488
489 IFQ_LOCK_ASSERT(ifq);
490
491 /* grab class set by classifier */
492 if ((m->m_flags & M_PKTHDR) == 0) {
493 /* should not happen */
494 printf("altq: packet for %s does not have pkthdr\n",
495 ifq->altq_ifp->if_xname);
496 m_freem(m);
497 return (ENOBUFS);
498 }
499
500 if ((t = pf_find_mtag(m)) != NULL) {
501 cl = clh_to_clp(pif, t->qid);
502 qid_hash = t->qid_hash;
503 }
504 if (cl == NULL) {
505 cl = pif->pif_default;
506 if (cl == NULL) {
507 m_freem(m);
508 return (ENOBUFS);
509 }
510 }
511 cl->cl_flags |= FARF_HAS_PACKETS;
512 cl->cl_pktattr = NULL;
513 len = m_pktlen(m);
514 if (fairq_addq(cl, m, qid_hash) != 0) {
515 /* drop occurred. mbuf was freed in fairq_addq. */
516 PKTCNTR_ADD(&cl->cl_dropcnt, len);
517 return (ENOBUFS);
518 }
519 IFQ_INC_LEN(ifq);
520
521 return (0);
522 }
523
524 /*
525 * fairq_dequeue is a dequeue function to be registered to
526 * (*altq_dequeue) in struct ifaltq.
527 *
528 * note: ALTDQ_POLL returns the next packet without removing the packet
529 * from the queue. ALTDQ_REMOVE is a normal dequeue operation.
530 * ALTDQ_REMOVE must return the same packet if called immediately
531 * after ALTDQ_POLL.
532 */
533 static struct mbuf *
fairq_dequeue(struct ifaltq * ifq,int op)534 fairq_dequeue(struct ifaltq *ifq, int op)
535 {
536 struct fairq_if *pif = (struct fairq_if *)ifq->altq_disc;
537 struct fairq_class *cl;
538 struct fairq_class *best_cl;
539 struct mbuf *best_m;
540 struct mbuf *m = NULL;
541 uint64_t cur_time = read_machclk();
542 int pri;
543 int hit_limit;
544
545 IFQ_LOCK_ASSERT(ifq);
546
547 if (IFQ_IS_EMPTY(ifq)) {
548 return (NULL);
549 }
550
551 if (pif->pif_poll_cache && op == ALTDQ_REMOVE) {
552 best_cl = pif->pif_poll_cache;
553 m = fairq_getq(best_cl, cur_time);
554 pif->pif_poll_cache = NULL;
555 if (m) {
556 IFQ_DEC_LEN(ifq);
557 PKTCNTR_ADD(&best_cl->cl_xmitcnt, m_pktlen(m));
558 return (m);
559 }
560 } else {
561 best_cl = NULL;
562 best_m = NULL;
563
564 for (pri = pif->pif_maxpri; pri >= 0; pri--) {
565 if ((cl = pif->pif_classes[pri]) == NULL)
566 continue;
567 if ((cl->cl_flags & FARF_HAS_PACKETS) == 0)
568 continue;
569 m = fairq_pollq(cl, cur_time, &hit_limit);
570 if (m == NULL) {
571 cl->cl_flags &= ~FARF_HAS_PACKETS;
572 continue;
573 }
574
575 /*
576 * Only override the best choice if we are under
577 * the BW limit.
578 */
579 if (hit_limit == 0 || best_cl == NULL) {
580 best_cl = cl;
581 best_m = m;
582 }
583
584 /*
585 * Remember the highest priority mbuf in case we
586 * do not find any lower priority mbufs.
587 */
588 if (hit_limit)
589 continue;
590 break;
591 }
592 if (op == ALTDQ_POLL) {
593 pif->pif_poll_cache = best_cl;
594 m = best_m;
595 } else if (best_cl) {
596 m = fairq_getq(best_cl, cur_time);
597 if (m != NULL) {
598 IFQ_DEC_LEN(ifq);
599 PKTCNTR_ADD(&best_cl->cl_xmitcnt, m_pktlen(m));
600 }
601 }
602 return (m);
603 }
604 return (NULL);
605 }
606
607 static int
fairq_addq(struct fairq_class * cl,struct mbuf * m,u_int32_t bucketid)608 fairq_addq(struct fairq_class *cl, struct mbuf *m, u_int32_t bucketid)
609 {
610 fairq_bucket_t *b;
611 u_int hindex;
612 uint64_t bw;
613
614 /*
615 * If the packet doesn't have any keep state put it on the end of
616 * our queue. XXX this can result in out of order delivery.
617 */
618 if (bucketid == 0) {
619 if (cl->cl_head)
620 b = cl->cl_head->prev;
621 else
622 b = &cl->cl_buckets[0];
623 } else {
624 hindex = bucketid & cl->cl_nbucket_mask;
625 b = &cl->cl_buckets[hindex];
626 }
627
628 /*
629 * Add the bucket to the end of the circular list of active buckets.
630 *
631 * As a special case we add the bucket to the beginning of the list
632 * instead of the end if it was not previously on the list and if
633 * its traffic is less then the hog level.
634 */
635 if (b->in_use == 0) {
636 b->in_use = 1;
637 if (cl->cl_head == NULL) {
638 cl->cl_head = b;
639 b->next = b;
640 b->prev = b;
641 } else {
642 b->next = cl->cl_head;
643 b->prev = cl->cl_head->prev;
644 b->prev->next = b;
645 b->next->prev = b;
646
647 if (b->bw_delta && cl->cl_hogs_m1) {
648 bw = b->bw_bytes * machclk_freq / b->bw_delta;
649 if (bw < cl->cl_hogs_m1)
650 cl->cl_head = b;
651 }
652 }
653 }
654
655 #ifdef ALTQ_RIO
656 if (cl->cl_qtype == Q_RIO)
657 return rio_addq((rio_t *)cl->cl_red, &b->queue, m, cl->cl_pktattr);
658 #endif
659 #ifdef ALTQ_RED
660 if (cl->cl_qtype == Q_RED)
661 return red_addq(cl->cl_red, &b->queue, m, cl->cl_pktattr);
662 #endif
663 #ifdef ALTQ_CODEL
664 if (cl->cl_qtype == Q_CODEL)
665 return codel_addq(cl->cl_codel, &b->queue, m);
666 #endif
667 if (qlen(&b->queue) >= qlimit(&b->queue)) {
668 m_freem(m);
669 return (-1);
670 }
671
672 if (cl->cl_flags & FARF_CLEARDSCP)
673 write_dsfield(m, cl->cl_pktattr, 0);
674
675 _addq(&b->queue, m);
676
677 return (0);
678 }
679
680 static struct mbuf *
fairq_getq(struct fairq_class * cl,uint64_t cur_time)681 fairq_getq(struct fairq_class *cl, uint64_t cur_time)
682 {
683 fairq_bucket_t *b;
684 struct mbuf *m;
685
686 b = fairq_selectq(cl, 0);
687 if (b == NULL)
688 m = NULL;
689 #ifdef ALTQ_RIO
690 else if (cl->cl_qtype == Q_RIO)
691 m = rio_getq((rio_t *)cl->cl_red, &b->queue);
692 #endif
693 #ifdef ALTQ_RED
694 else if (cl->cl_qtype == Q_RED)
695 m = red_getq(cl->cl_red, &b->queue);
696 #endif
697 #ifdef ALTQ_CODEL
698 else if (cl->cl_qtype == Q_CODEL)
699 m = codel_getq(cl->cl_codel, &b->queue);
700 #endif
701 else
702 m = _getq(&b->queue);
703
704 /*
705 * Calculate the BW change
706 */
707 if (m != NULL) {
708 uint64_t delta;
709
710 /*
711 * Per-class bandwidth calculation
712 */
713 delta = (cur_time - cl->cl_last_time);
714 if (delta > machclk_freq * 8)
715 delta = machclk_freq * 8;
716 cl->cl_bw_delta += delta;
717 cl->cl_bw_bytes += m->m_pkthdr.len;
718 cl->cl_last_time = cur_time;
719 cl->cl_bw_delta -= cl->cl_bw_delta >> 3;
720 cl->cl_bw_bytes -= cl->cl_bw_bytes >> 3;
721
722 /*
723 * Per-bucket bandwidth calculation
724 */
725 delta = (cur_time - b->last_time);
726 if (delta > machclk_freq * 8)
727 delta = machclk_freq * 8;
728 b->bw_delta += delta;
729 b->bw_bytes += m->m_pkthdr.len;
730 b->last_time = cur_time;
731 b->bw_delta -= b->bw_delta >> 3;
732 b->bw_bytes -= b->bw_bytes >> 3;
733 }
734 return(m);
735 }
736
737 /*
738 * Figure out what the next packet would be if there were no limits. If
739 * this class hits its bandwidth limit *hit_limit is set to no-zero, otherwise
740 * it is set to 0. A non-NULL mbuf is returned either way.
741 */
742 static struct mbuf *
fairq_pollq(struct fairq_class * cl,uint64_t cur_time,int * hit_limit)743 fairq_pollq(struct fairq_class *cl, uint64_t cur_time, int *hit_limit)
744 {
745 fairq_bucket_t *b;
746 struct mbuf *m;
747 uint64_t delta;
748 uint64_t bw;
749
750 *hit_limit = 0;
751 b = fairq_selectq(cl, 1);
752 if (b == NULL)
753 return(NULL);
754 m = qhead(&b->queue);
755
756 /*
757 * Did this packet exceed the class bandwidth? Calculate the
758 * bandwidth component of the packet.
759 *
760 * - Calculate bytes per second
761 */
762 delta = cur_time - cl->cl_last_time;
763 if (delta > machclk_freq * 8)
764 delta = machclk_freq * 8;
765 cl->cl_bw_delta += delta;
766 cl->cl_last_time = cur_time;
767 if (cl->cl_bw_delta) {
768 bw = cl->cl_bw_bytes * machclk_freq / cl->cl_bw_delta;
769
770 if (bw > cl->cl_bandwidth)
771 *hit_limit = 1;
772 #ifdef ALTQ_DEBUG
773 printf("BW %6ju relative to %6u %d queue %p\n",
774 (uintmax_t)bw, cl->cl_bandwidth, *hit_limit, b);
775 #endif
776 }
777 return(m);
778 }
779
780 /*
781 * Locate the next queue we want to pull a packet out of. This code
782 * is also responsible for removing empty buckets from the circular list.
783 */
784 static
785 fairq_bucket_t *
fairq_selectq(struct fairq_class * cl,int ispoll)786 fairq_selectq(struct fairq_class *cl, int ispoll)
787 {
788 fairq_bucket_t *b;
789 uint64_t bw;
790
791 if (ispoll == 0 && cl->cl_polled) {
792 b = cl->cl_polled;
793 cl->cl_polled = NULL;
794 return(b);
795 }
796
797 while ((b = cl->cl_head) != NULL) {
798 /*
799 * Remove empty queues from consideration
800 */
801 if (qempty(&b->queue)) {
802 b->in_use = 0;
803 cl->cl_head = b->next;
804 if (cl->cl_head == b) {
805 cl->cl_head = NULL;
806 } else {
807 b->next->prev = b->prev;
808 b->prev->next = b->next;
809 }
810 continue;
811 }
812
813 /*
814 * Advance the round robin. Queues with bandwidths less
815 * then the hog bandwidth are allowed to burst.
816 */
817 if (cl->cl_hogs_m1 == 0) {
818 cl->cl_head = b->next;
819 } else if (b->bw_delta) {
820 bw = b->bw_bytes * machclk_freq / b->bw_delta;
821 if (bw >= cl->cl_hogs_m1) {
822 cl->cl_head = b->next;
823 }
824 /*
825 * XXX TODO -
826 */
827 }
828
829 /*
830 * Return bucket b.
831 */
832 break;
833 }
834 if (ispoll)
835 cl->cl_polled = b;
836 return(b);
837 }
838
839 static void
fairq_purgeq(struct fairq_class * cl)840 fairq_purgeq(struct fairq_class *cl)
841 {
842 fairq_bucket_t *b;
843 struct mbuf *m;
844
845 while ((b = fairq_selectq(cl, 0)) != NULL) {
846 while ((m = _getq(&b->queue)) != NULL) {
847 PKTCNTR_ADD(&cl->cl_dropcnt, m_pktlen(m));
848 m_freem(m);
849 }
850 ASSERT(qlen(&b->queue) == 0);
851 }
852 }
853
854 static void
get_class_stats(struct fairq_classstats * sp,struct fairq_class * cl)855 get_class_stats(struct fairq_classstats *sp, struct fairq_class *cl)
856 {
857 fairq_bucket_t *b;
858
859 sp->class_handle = cl->cl_handle;
860 sp->qlimit = cl->cl_qlimit;
861 sp->xmit_cnt = cl->cl_xmitcnt;
862 sp->drop_cnt = cl->cl_dropcnt;
863 sp->qtype = cl->cl_qtype;
864 sp->qlength = 0;
865
866 if (cl->cl_head) {
867 b = cl->cl_head;
868 do {
869 sp->qlength += qlen(&b->queue);
870 b = b->next;
871 } while (b != cl->cl_head);
872 }
873
874 #ifdef ALTQ_RED
875 if (cl->cl_qtype == Q_RED)
876 red_getstats(cl->cl_red, &sp->red[0]);
877 #endif
878 #ifdef ALTQ_RIO
879 if (cl->cl_qtype == Q_RIO)
880 rio_getstats((rio_t *)cl->cl_red, &sp->red[0]);
881 #endif
882 #ifdef ALTQ_CODEL
883 if (cl->cl_qtype == Q_CODEL)
884 codel_getstats(cl->cl_codel, &sp->codel);
885 #endif
886 }
887
888 /* convert a class handle to the corresponding class pointer */
889 static struct fairq_class *
clh_to_clp(struct fairq_if * pif,uint32_t chandle)890 clh_to_clp(struct fairq_if *pif, uint32_t chandle)
891 {
892 struct fairq_class *cl;
893 int idx;
894
895 if (chandle == 0)
896 return (NULL);
897
898 for (idx = pif->pif_maxpri; idx >= 0; idx--)
899 if ((cl = pif->pif_classes[idx]) != NULL &&
900 cl->cl_handle == chandle)
901 return (cl);
902
903 return (NULL);
904 }
905
906 #endif /* ALTQ_FAIRQ */
907