1 /* $NetBSD: bpf.c,v 1.258 2024/10/20 14:03:51 mlelstv Exp $ */
2
3 /*
4 * Copyright (c) 1990, 1991, 1993
5 * The Regents of the University of California. All rights reserved.
6 *
7 * This code is derived from the Stanford/CMU enet packet filter,
8 * (net/enet.c) distributed as part of 4.3BSD, and code contributed
9 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence
10 * Berkeley Laboratory.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 * 3. Neither the name of the University nor the names of its contributors
21 * may be used to endorse or promote products derived from this software
22 * without specific prior written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 *
36 * @(#)bpf.c 8.4 (Berkeley) 1/9/95
37 * static char rcsid[] =
38 * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp ";
39 */
40
41 #include <sys/cdefs.h>
42 __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.258 2024/10/20 14:03:51 mlelstv Exp $");
43
44 #if defined(_KERNEL_OPT)
45 #include "opt_bpf.h"
46 #include "sl.h"
47 #include "opt_net_mpsafe.h"
48 #endif
49
50 #include <sys/param.h>
51
52 #include <sys/atomic.h>
53 #include <sys/buf.h>
54 #include <sys/conf.h>
55 #include <sys/cpu.h>
56 #include <sys/errno.h>
57 #include <sys/file.h>
58 #include <sys/filedesc.h>
59 #include <sys/ioctl.h>
60 #include <sys/kauth.h>
61 #include <sys/kernel.h>
62 #include <sys/lwp.h>
63 #include <sys/mbuf.h>
64 #include <sys/module.h>
65 #include <sys/percpu.h>
66 #include <sys/poll.h>
67 #include <sys/proc.h>
68 #include <sys/protosw.h>
69 #include <sys/pserialize.h>
70 #include <sys/queue.h>
71 #include <sys/socket.h>
72 #include <sys/stat.h>
73 #include <sys/sysctl.h>
74 #include <sys/syslog.h>
75 #include <sys/systm.h>
76 #include <sys/time.h>
77 #include <sys/tty.h>
78 #include <sys/uio.h>
79 #include <sys/vnode.h>
80 #include <sys/xcall.h>
81
82 #include <net/bpf.h>
83 #include <net/bpfdesc.h>
84 #include <net/bpfjit.h>
85 #include <net/if.h>
86 #include <net/if_arc.h>
87 #include <net/if_ether.h>
88 #include <net/if_types.h>
89 #include <net/slip.h>
90
91 #include <netinet/if_inarp.h>
92 #include <netinet/in.h>
93
94 #include <compat/sys/sockio.h>
95
96 #ifndef BPF_BUFSIZE
97 /*
98 * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet
99 * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k).
100 */
101 # define BPF_BUFSIZE 32768
102 #endif
103
104 #define PRINET 26 /* interruptible */
105
106 /*
107 * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able.
108 * XXX the default values should be computed dynamically based
109 * on available memory size and available mbuf clusters.
110 */
111 static int bpf_bufsize = BPF_BUFSIZE;
112 static int bpf_maxbufsize = BPF_DFLTBUFSIZE; /* XXX set dynamically, see above */
113 static bool bpf_jit = false;
114
115 struct bpfjit_ops bpfjit_module_ops = {
116 .bj_generate_code = NULL,
117 .bj_free_code = NULL
118 };
119
120 /*
121 * Global BPF statistics returned by net.bpf.stats sysctl.
122 */
123 static struct percpu *bpf_gstats_percpu; /* struct bpf_stat */
124
125 #define BPF_STATINC(id) \
126 { \
127 struct bpf_stat *__stats = \
128 percpu_getref(bpf_gstats_percpu); \
129 __stats->bs_##id++; \
130 percpu_putref(bpf_gstats_percpu); \
131 }
132
133 /*
134 * Locking notes:
135 * - bpf_mtx (adaptive mutex) protects:
136 * - Gobal lists: bpf_iflist and bpf_dlist
137 * - struct bpf_if
138 * - bpf_close
139 * - bpf_psz (pserialize)
140 * - struct bpf_d has two mutexes:
141 * - bd_buf_mtx (spin mutex) protects the buffers that can be accessed
142 * on packet tapping
143 * - bd_mtx (adaptive mutex) protects member variables other than the buffers
144 * - Locking order: bpf_mtx => bpf_d#bd_mtx => bpf_d#bd_buf_mtx
145 * - struct bpf_d obtained via fp->f_bpf in bpf_read and bpf_write is
146 * never freed because struct bpf_d is only freed in bpf_close and
147 * bpf_close never be called while executing bpf_read and bpf_write
148 * - A filter that is assigned to bpf_d can be replaced with another filter
149 * while tapping packets, so it needs to be done atomically
150 * - struct bpf_d is iterated on bpf_dlist with psz
151 * - struct bpf_if is iterated on bpf_iflist with psz or psref
152 */
153 /*
154 * Use a mutex to avoid a race condition between gathering the stats/peers
155 * and opening/closing the device.
156 */
157 static kmutex_t bpf_mtx;
158
159 static struct psref_class *bpf_psref_class __read_mostly;
160 static pserialize_t bpf_psz;
161
162 static inline void
bpf_if_acquire(struct bpf_if * bp,struct psref * psref)163 bpf_if_acquire(struct bpf_if *bp, struct psref *psref)
164 {
165
166 psref_acquire(psref, &bp->bif_psref, bpf_psref_class);
167 }
168
169 static inline void
bpf_if_release(struct bpf_if * bp,struct psref * psref)170 bpf_if_release(struct bpf_if *bp, struct psref *psref)
171 {
172
173 psref_release(psref, &bp->bif_psref, bpf_psref_class);
174 }
175
176 /*
177 * bpf_iflist is the list of interfaces; each corresponds to an ifnet
178 * bpf_dtab holds the descriptors, indexed by minor device #
179 */
180 static struct pslist_head bpf_iflist;
181 static struct pslist_head bpf_dlist;
182
183 /* Macros for bpf_d on bpf_dlist */
184 #define BPF_DLIST_WRITER_INSERT_HEAD(__d) \
185 PSLIST_WRITER_INSERT_HEAD(&bpf_dlist, (__d), bd_bpf_dlist_entry)
186 #define BPF_DLIST_READER_FOREACH(__d) \
187 PSLIST_READER_FOREACH((__d), &bpf_dlist, struct bpf_d, \
188 bd_bpf_dlist_entry)
189 #define BPF_DLIST_WRITER_FOREACH(__d) \
190 PSLIST_WRITER_FOREACH((__d), &bpf_dlist, struct bpf_d, \
191 bd_bpf_dlist_entry)
192 #define BPF_DLIST_ENTRY_INIT(__d) \
193 PSLIST_ENTRY_INIT((__d), bd_bpf_dlist_entry)
194 #define BPF_DLIST_WRITER_REMOVE(__d) \
195 PSLIST_WRITER_REMOVE((__d), bd_bpf_dlist_entry)
196 #define BPF_DLIST_ENTRY_DESTROY(__d) \
197 PSLIST_ENTRY_DESTROY((__d), bd_bpf_dlist_entry)
198
199 /* Macros for bpf_if on bpf_iflist */
200 #define BPF_IFLIST_WRITER_INSERT_HEAD(__bp) \
201 PSLIST_WRITER_INSERT_HEAD(&bpf_iflist, (__bp), bif_iflist_entry)
202 #define BPF_IFLIST_READER_FOREACH(__bp) \
203 PSLIST_READER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \
204 bif_iflist_entry)
205 #define BPF_IFLIST_WRITER_FOREACH(__bp) \
206 PSLIST_WRITER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \
207 bif_iflist_entry)
208 #define BPF_IFLIST_WRITER_REMOVE(__bp) \
209 PSLIST_WRITER_REMOVE((__bp), bif_iflist_entry)
210 #define BPF_IFLIST_ENTRY_INIT(__bp) \
211 PSLIST_ENTRY_INIT((__bp), bif_iflist_entry)
212 #define BPF_IFLIST_ENTRY_DESTROY(__bp) \
213 PSLIST_ENTRY_DESTROY((__bp), bif_iflist_entry)
214
215 /* Macros for bpf_d on bpf_if#bif_dlist_pslist */
216 #define BPFIF_DLIST_READER_FOREACH(__d, __bp) \
217 PSLIST_READER_FOREACH((__d), &(__bp)->bif_dlist_head, struct bpf_d, \
218 bd_bif_dlist_entry)
219 #define BPFIF_DLIST_WRITER_INSERT_HEAD(__bp, __d) \
220 PSLIST_WRITER_INSERT_HEAD(&(__bp)->bif_dlist_head, (__d), \
221 bd_bif_dlist_entry)
222 #define BPFIF_DLIST_WRITER_REMOVE(__d) \
223 PSLIST_WRITER_REMOVE((__d), bd_bif_dlist_entry)
224 #define BPFIF_DLIST_ENTRY_INIT(__d) \
225 PSLIST_ENTRY_INIT((__d), bd_bif_dlist_entry)
226 #define BPFIF_DLIST_READER_EMPTY(__bp) \
227 (PSLIST_READER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \
228 bd_bif_dlist_entry) == NULL)
229 #define BPFIF_DLIST_WRITER_EMPTY(__bp) \
230 (PSLIST_WRITER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \
231 bd_bif_dlist_entry) == NULL)
232 #define BPFIF_DLIST_ENTRY_DESTROY(__d) \
233 PSLIST_ENTRY_DESTROY((__d), bd_bif_dlist_entry)
234
235 static int bpf_allocbufs(struct bpf_d *);
236 static u_int bpf_xfilter(struct bpf_filter **, void *, u_int, u_int);
237 static void bpf_deliver(struct bpf_if *,
238 void *(*cpfn)(void *, const void *, size_t),
239 void *, u_int, u_int, const u_int);
240 static void bpf_freed(struct bpf_d *);
241 static void bpf_free_filter(struct bpf_filter *);
242 static void bpf_ifname(struct ifnet *, struct ifreq *);
243 static void *bpf_mcpy(void *, const void *, size_t);
244 static int bpf_movein(struct ifnet *, struct uio *, int, uint64_t,
245 struct mbuf **, struct sockaddr *,
246 struct bpf_filter **);
247 static void bpf_attachd(struct bpf_d *, struct bpf_if *);
248 static void bpf_detachd(struct bpf_d *);
249 static int bpf_setif(struct bpf_d *, struct ifreq *);
250 static int bpf_setf(struct bpf_d *, struct bpf_program *, u_long);
251 static void bpf_timed_out(void *);
252 static inline void
253 bpf_wakeup(struct bpf_d *);
254 static int bpf_hdrlen(struct bpf_d *);
255 static void catchpacket(struct bpf_d *, u_char *, u_int, u_int,
256 void *(*)(void *, const void *, size_t),
257 struct timespec *);
258 static void reset_d(struct bpf_d *);
259 static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *);
260 static int bpf_setdlt(struct bpf_d *, u_int);
261
262 static int bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t,
263 int);
264 static int bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t,
265 int);
266 static int bpf_ioctl(struct file *, u_long, void *);
267 static int bpf_poll(struct file *, int);
268 static int bpf_stat(struct file *, struct stat *);
269 static int bpf_close(struct file *);
270 static int bpf_kqfilter(struct file *, struct knote *);
271 static void bpf_softintr(void *);
272
273 static const struct fileops bpf_fileops = {
274 .fo_name = "bpf",
275 .fo_read = bpf_read,
276 .fo_write = bpf_write,
277 .fo_ioctl = bpf_ioctl,
278 .fo_fcntl = fnullop_fcntl,
279 .fo_poll = bpf_poll,
280 .fo_stat = bpf_stat,
281 .fo_close = bpf_close,
282 .fo_kqfilter = bpf_kqfilter,
283 .fo_restart = fnullop_restart,
284 };
285
286 dev_type_open(bpfopen);
287
288 const struct cdevsw bpf_cdevsw = {
289 .d_open = bpfopen,
290 .d_close = noclose,
291 .d_read = noread,
292 .d_write = nowrite,
293 .d_ioctl = noioctl,
294 .d_stop = nostop,
295 .d_tty = notty,
296 .d_poll = nopoll,
297 .d_mmap = nommap,
298 .d_kqfilter = nokqfilter,
299 .d_discard = nodiscard,
300 .d_flag = D_OTHER | D_MPSAFE
301 };
302
303 bpfjit_func_t
bpf_jit_generate(bpf_ctx_t * bc,void * code,size_t size)304 bpf_jit_generate(bpf_ctx_t *bc, void *code, size_t size)
305 {
306 struct bpfjit_ops *ops = &bpfjit_module_ops;
307 bpfjit_func_t (*generate_code)(const bpf_ctx_t *,
308 const struct bpf_insn *, size_t);
309
310 generate_code = atomic_load_acquire(&ops->bj_generate_code);
311 if (generate_code != NULL) {
312 return generate_code(bc, code, size);
313 }
314 return NULL;
315 }
316
317 void
bpf_jit_freecode(bpfjit_func_t jcode)318 bpf_jit_freecode(bpfjit_func_t jcode)
319 {
320 KASSERT(bpfjit_module_ops.bj_free_code != NULL);
321 bpfjit_module_ops.bj_free_code(jcode);
322 }
323
324 static int
bpf_movein(struct ifnet * ifp,struct uio * uio,int linktype,uint64_t mtu,struct mbuf ** mp,struct sockaddr * sockp,struct bpf_filter ** wfilter)325 bpf_movein(struct ifnet *ifp, struct uio *uio, int linktype, uint64_t mtu,
326 struct mbuf **mp, struct sockaddr *sockp, struct bpf_filter **wfilter)
327 {
328 struct mbuf *m, *m0, *n;
329 int error;
330 size_t len;
331 size_t hlen;
332 size_t align;
333 u_int slen;
334
335 /*
336 * Build a sockaddr based on the data link layer type.
337 * We do this at this level because the ethernet header
338 * is copied directly into the data field of the sockaddr.
339 * In the case of SLIP, there is no header and the packet
340 * is forwarded as is.
341 * Also, we are careful to leave room at the front of the mbuf
342 * for the link level header.
343 */
344 switch (linktype) {
345
346 case DLT_SLIP:
347 sockp->sa_family = AF_INET;
348 hlen = 0;
349 align = 0;
350 break;
351
352 case DLT_PPP:
353 sockp->sa_family = AF_UNSPEC;
354 hlen = 0;
355 align = 0;
356 break;
357
358 case DLT_EN10MB:
359 sockp->sa_family = AF_UNSPEC;
360 /* XXX Would MAXLINKHDR be better? */
361 /* 6(dst)+6(src)+2(type) */
362 hlen = sizeof(struct ether_header);
363 align = 2;
364 break;
365
366 case DLT_ARCNET:
367 sockp->sa_family = AF_UNSPEC;
368 hlen = ARC_HDRLEN;
369 align = 5;
370 break;
371
372 case DLT_FDDI:
373 sockp->sa_family = AF_LINK;
374 /* XXX 4(FORMAC)+6(dst)+6(src) */
375 hlen = 16;
376 align = 0;
377 break;
378
379 case DLT_ECONET:
380 sockp->sa_family = AF_UNSPEC;
381 hlen = 6;
382 align = 2;
383 break;
384
385 case DLT_NULL:
386 sockp->sa_family = AF_UNSPEC;
387 if (ifp->if_type == IFT_LOOP) {
388 /* Set here to apply the following validations */
389 hlen = sizeof(uint32_t);
390 } else
391 hlen = 0;
392 align = 0;
393 break;
394
395 default:
396 return (EIO);
397 }
398
399 len = uio->uio_resid;
400 /*
401 * If there aren't enough bytes for a link level header or the
402 * packet length exceeds the interface mtu, return an error.
403 */
404 if (len - hlen > mtu)
405 return (EMSGSIZE);
406
407 m0 = m = m_gethdr(M_WAIT, MT_DATA);
408 m_reset_rcvif(m);
409 m->m_pkthdr.len = (int)(len - hlen);
410 if (len + align > MHLEN) {
411 m_clget(m, M_WAIT);
412 if ((m->m_flags & M_EXT) == 0) {
413 error = ENOBUFS;
414 goto bad;
415 }
416 }
417
418 /* Ensure the data is properly aligned */
419 if (align > 0)
420 m->m_data += align;
421
422 for (;;) {
423 len = M_TRAILINGSPACE(m);
424 if (len > uio->uio_resid)
425 len = uio->uio_resid;
426 error = uiomove(mtod(m, void *), len, uio);
427 if (error)
428 goto bad;
429 m->m_len = len;
430
431 if (uio->uio_resid == 0)
432 break;
433
434 n = m_get(M_WAIT, MT_DATA);
435 m_clget(n, M_WAIT); /* if fails, there is no problem */
436 m->m_next = n;
437 m = n;
438 }
439
440 slen = bpf_xfilter(wfilter, mtod(m, u_char *), len, len);
441 if (slen == 0) {
442 error = EPERM;
443 goto bad;
444 }
445
446 if (hlen != 0) {
447 if (linktype == DLT_NULL && ifp->if_type == IFT_LOOP) {
448 uint32_t af;
449 /* the link header indicates the address family */
450 memcpy(&af, mtod(m0, void *), sizeof(af));
451 sockp->sa_family = af;
452 } else {
453 /* move link level header in the top of mbuf to sa_data */
454 memcpy(sockp->sa_data, mtod(m0, void *), hlen);
455 }
456 m0->m_data += hlen;
457 m0->m_len -= hlen;
458 }
459
460 m_claimm(m, ifp->if_mowner);
461
462 *mp = m0;
463 return (0);
464
465 bad:
466 m_freem(m0);
467 return (error);
468 }
469
470 /*
471 * Attach file to the bpf interface, i.e. make d listen on bp.
472 */
473 static void
bpf_attachd(struct bpf_d * d,struct bpf_if * bp)474 bpf_attachd(struct bpf_d *d, struct bpf_if *bp)
475 {
476 struct bpf_event_tracker *t;
477
478 KASSERT(mutex_owned(&bpf_mtx));
479 KASSERT(mutex_owned(d->bd_mtx));
480 /*
481 * Point d at bp, and add d to the interface's list of listeners.
482 * Finally, point the driver's bpf cookie at the interface so
483 * it will divert packets to bpf.
484 */
485 d->bd_bif = bp;
486 BPFIF_DLIST_WRITER_INSERT_HEAD(bp, d);
487
488 *bp->bif_driverp = bp;
489
490 SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) {
491 t->bet_notify(bp, bp->bif_ifp, bp->bif_dlt,
492 BPF_TRACK_EVENT_ATTACH);
493 }
494 }
495
496 /*
497 * Detach a file from its interface.
498 */
499 static void
bpf_detachd(struct bpf_d * d)500 bpf_detachd(struct bpf_d *d)
501 {
502 struct bpf_if *bp;
503 struct bpf_event_tracker *t;
504
505 KASSERT(mutex_owned(&bpf_mtx));
506 KASSERT(mutex_owned(d->bd_mtx));
507
508 bp = d->bd_bif;
509 /*
510 * Check if this descriptor had requested promiscuous mode.
511 * If so, turn it off.
512 */
513 if (d->bd_promisc) {
514 int error __diagused;
515
516 d->bd_promisc = 0;
517 /*
518 * Take device out of promiscuous mode. Since we were
519 * able to enter promiscuous mode, we should be able
520 * to turn it off. But we can get an error if
521 * the interface was configured down, so only panic
522 * if we don't get an unexpected error.
523 */
524 KERNEL_LOCK_UNLESS_NET_MPSAFE();
525 error = ifpromisc(bp->bif_ifp, 0);
526 KERNEL_UNLOCK_UNLESS_NET_MPSAFE();
527 #ifdef DIAGNOSTIC
528 if (error)
529 printf("%s: ifpromisc failed: %d", __func__, error);
530 #endif
531 }
532
533 /* Remove d from the interface's descriptor list. */
534 BPFIF_DLIST_WRITER_REMOVE(d);
535
536 pserialize_perform(bpf_psz);
537
538 if (BPFIF_DLIST_WRITER_EMPTY(bp)) {
539 /*
540 * Let the driver know that there are no more listeners.
541 */
542 *d->bd_bif->bif_driverp = NULL;
543 }
544
545 d->bd_bif = NULL;
546
547 SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) {
548 t->bet_notify(bp, bp->bif_ifp, bp->bif_dlt,
549 BPF_TRACK_EVENT_DETACH);
550 }
551 }
552
553 static void
bpf_init(void)554 bpf_init(void)
555 {
556
557 mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE);
558 bpf_psz = pserialize_create();
559 bpf_psref_class = psref_class_create("bpf", IPL_SOFTNET);
560
561 PSLIST_INIT(&bpf_iflist);
562 PSLIST_INIT(&bpf_dlist);
563
564 bpf_gstats_percpu = percpu_alloc(sizeof(struct bpf_stat));
565
566 return;
567 }
568
569 /*
570 * bpfilterattach() is called at boot time. We don't need to do anything
571 * here, since any initialization will happen as part of module init code.
572 */
573 /* ARGSUSED */
574 void
bpfilterattach(int n)575 bpfilterattach(int n)
576 {
577
578 }
579
580 /*
581 * Open ethernet device. Clones.
582 */
583 /* ARGSUSED */
584 int
bpfopen(dev_t dev,int flag,int mode,struct lwp * l)585 bpfopen(dev_t dev, int flag, int mode, struct lwp *l)
586 {
587 struct bpf_d *d;
588 struct file *fp;
589 int error, fd;
590
591 /* falloc() will fill in the descriptor for us. */
592 if ((error = fd_allocfile(&fp, &fd)) != 0)
593 return error;
594
595 d = kmem_zalloc(sizeof(*d), KM_SLEEP);
596 d->bd_bufsize = bpf_bufsize;
597 d->bd_direction = BPF_D_INOUT;
598 d->bd_feedback = 0;
599 d->bd_pid = l->l_proc->p_pid;
600 #ifdef _LP64
601 if (curproc->p_flag & PK_32)
602 d->bd_compat32 = 1;
603 #endif
604 getnanotime(&d->bd_btime);
605 d->bd_atime = d->bd_mtime = d->bd_btime;
606 callout_init(&d->bd_callout, CALLOUT_MPSAFE);
607 selinit(&d->bd_sel);
608 d->bd_sih = softint_establish(SOFTINT_CLOCK, bpf_softintr, d);
609 d->bd_jitcode = NULL;
610 d->bd_rfilter = NULL;
611 d->bd_wfilter = NULL;
612 d->bd_locked = 0;
613 BPF_DLIST_ENTRY_INIT(d);
614 BPFIF_DLIST_ENTRY_INIT(d);
615 d->bd_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET);
616 d->bd_buf_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
617 cv_init(&d->bd_cv, "bpf");
618
619 mutex_enter(&bpf_mtx);
620 BPF_DLIST_WRITER_INSERT_HEAD(d);
621 mutex_exit(&bpf_mtx);
622
623 return fd_clone(fp, fd, flag, &bpf_fileops, d);
624 }
625
626 /*
627 * Close the descriptor by detaching it from its interface,
628 * deallocating its buffers, and marking it free.
629 */
630 /* ARGSUSED */
631 static int
bpf_close(struct file * fp)632 bpf_close(struct file *fp)
633 {
634 struct bpf_d *d;
635
636 mutex_enter(&bpf_mtx);
637
638 if ((d = fp->f_bpf) == NULL) {
639 mutex_exit(&bpf_mtx);
640 return 0;
641 }
642
643 /*
644 * Refresh the PID associated with this bpf file.
645 */
646 d->bd_pid = curproc->p_pid;
647
648 mutex_enter(d->bd_buf_mtx);
649 if (d->bd_state == BPF_WAITING)
650 callout_halt(&d->bd_callout, d->bd_buf_mtx);
651 d->bd_state = BPF_IDLE;
652 mutex_exit(d->bd_buf_mtx);
653 mutex_enter(d->bd_mtx);
654 if (d->bd_bif)
655 bpf_detachd(d);
656 mutex_exit(d->bd_mtx);
657
658 BPF_DLIST_WRITER_REMOVE(d);
659
660 pserialize_perform(bpf_psz);
661 mutex_exit(&bpf_mtx);
662
663 BPFIF_DLIST_ENTRY_DESTROY(d);
664 BPF_DLIST_ENTRY_DESTROY(d);
665 fp->f_bpf = NULL;
666 bpf_freed(d);
667 callout_destroy(&d->bd_callout);
668 seldestroy(&d->bd_sel);
669 softint_disestablish(d->bd_sih);
670 mutex_obj_free(d->bd_mtx);
671 mutex_obj_free(d->bd_buf_mtx);
672 cv_destroy(&d->bd_cv);
673
674 kmem_free(d, sizeof(*d));
675
676 return (0);
677 }
678
679 /*
680 * Rotate the packet buffers in descriptor d. Move the store buffer
681 * into the hold slot, and the free buffer into the store slot.
682 * Zero the length of the new store buffer.
683 */
684 #define ROTATE_BUFFERS(d) \
685 (d)->bd_hbuf = (d)->bd_sbuf; \
686 (d)->bd_hlen = (d)->bd_slen; \
687 (d)->bd_sbuf = (d)->bd_fbuf; \
688 (d)->bd_slen = 0; \
689 (d)->bd_fbuf = NULL;
690 /*
691 * bpfread - read next chunk of packets from buffers
692 */
693 static int
bpf_read(struct file * fp,off_t * offp,struct uio * uio,kauth_cred_t cred,int flags)694 bpf_read(struct file *fp, off_t *offp, struct uio *uio,
695 kauth_cred_t cred, int flags)
696 {
697 struct bpf_d *d = fp->f_bpf;
698 int timed_out;
699 int error;
700
701 /*
702 * Refresh the PID associated with this bpf file.
703 */
704 d->bd_pid = curproc->p_pid;
705
706 getnanotime(&d->bd_atime);
707 /*
708 * Restrict application to use a buffer the same size as
709 * the kernel buffers.
710 */
711 if (uio->uio_resid != d->bd_bufsize)
712 return (EINVAL);
713
714 mutex_enter(d->bd_buf_mtx);
715 if (d->bd_state == BPF_WAITING)
716 callout_halt(&d->bd_callout, d->bd_buf_mtx);
717 timed_out = (d->bd_state == BPF_TIMED_OUT);
718 d->bd_state = BPF_IDLE;
719 mutex_exit(d->bd_buf_mtx);
720 /*
721 * If the hold buffer is empty, then do a timed sleep, which
722 * ends when the timeout expires or when enough packets
723 * have arrived to fill the store buffer.
724 */
725 mutex_enter(d->bd_buf_mtx);
726 while (d->bd_hbuf == NULL) {
727 if (fp->f_flag & FNONBLOCK) {
728 if (d->bd_slen == 0) {
729 error = EWOULDBLOCK;
730 goto out;
731 }
732 ROTATE_BUFFERS(d);
733 break;
734 }
735
736 if ((d->bd_immediate || timed_out) && d->bd_slen != 0) {
737 /*
738 * A packet(s) either arrived since the previous
739 * read or arrived while we were asleep.
740 * Rotate the buffers and return what's here.
741 */
742 ROTATE_BUFFERS(d);
743 break;
744 }
745
746 error = cv_timedwait_sig(&d->bd_cv, d->bd_buf_mtx, d->bd_rtout);
747
748 if (error == EINTR || error == ERESTART)
749 goto out;
750
751 if (error == EWOULDBLOCK) {
752 /*
753 * On a timeout, return what's in the buffer,
754 * which may be nothing. If there is something
755 * in the store buffer, we can rotate the buffers.
756 */
757 if (d->bd_hbuf)
758 /*
759 * We filled up the buffer in between
760 * getting the timeout and arriving
761 * here, so we don't need to rotate.
762 */
763 break;
764
765 if (d->bd_slen == 0) {
766 error = 0;
767 goto out;
768 }
769 ROTATE_BUFFERS(d);
770 break;
771 }
772 if (error != 0)
773 goto out;
774 }
775 /*
776 * At this point, we know we have something in the hold slot.
777 */
778 mutex_exit(d->bd_buf_mtx);
779
780 /*
781 * Move data from hold buffer into user space.
782 * We know the entire buffer is transferred since
783 * we checked above that the read buffer is bpf_bufsize bytes.
784 */
785 error = uiomove(d->bd_hbuf, d->bd_hlen, uio);
786
787 mutex_enter(d->bd_buf_mtx);
788 d->bd_fbuf = d->bd_hbuf;
789 d->bd_hbuf = NULL;
790 d->bd_hlen = 0;
791 out:
792 mutex_exit(d->bd_buf_mtx);
793 return (error);
794 }
795
796 /*
797 * If there are processes sleeping on this descriptor, wake them up.
798 */
799 static inline void
bpf_wakeup(struct bpf_d * d)800 bpf_wakeup(struct bpf_d *d)
801 {
802
803 KASSERT(mutex_owned(d->bd_buf_mtx));
804
805 cv_broadcast(&d->bd_cv);
806
807 if (d->bd_async)
808 softint_schedule(d->bd_sih);
809 selnotify(&d->bd_sel, 0, NOTE_SUBMIT);
810 }
811
812 static void
bpf_softintr(void * cookie)813 bpf_softintr(void *cookie)
814 {
815 struct bpf_d *d;
816
817 d = cookie;
818 if (d->bd_async)
819 fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL);
820 }
821
822 static void
bpf_timed_out(void * arg)823 bpf_timed_out(void *arg)
824 {
825 struct bpf_d *d = arg;
826
827 mutex_enter(d->bd_buf_mtx);
828 if (d->bd_state == BPF_WAITING) {
829 d->bd_state = BPF_TIMED_OUT;
830 if (d->bd_slen != 0)
831 bpf_wakeup(d);
832 }
833 mutex_exit(d->bd_buf_mtx);
834 }
835
836 static int
bpf_write(struct file * fp,off_t * offp,struct uio * uio,kauth_cred_t cred,int flags)837 bpf_write(struct file *fp, off_t *offp, struct uio *uio,
838 kauth_cred_t cred, int flags)
839 {
840 struct bpf_d *d = fp->f_bpf;
841 struct bpf_if *bp;
842 struct ifnet *ifp;
843 struct mbuf *m, *mc;
844 int error;
845 static struct sockaddr_storage dst;
846 struct psref psref;
847 int bound;
848
849 /*
850 * Refresh the PID associated with this bpf file.
851 */
852 d->bd_pid = curproc->p_pid;
853
854 m = NULL; /* XXX gcc */
855
856 bound = curlwp_bind();
857 mutex_enter(d->bd_mtx);
858 bp = d->bd_bif;
859 if (bp == NULL) {
860 mutex_exit(d->bd_mtx);
861 error = ENXIO;
862 goto out_bindx;
863 }
864 bpf_if_acquire(bp, &psref);
865 mutex_exit(d->bd_mtx);
866
867 getnanotime(&d->bd_mtime);
868
869 ifp = bp->bif_ifp;
870 if (if_is_deactivated(ifp)) {
871 error = ENXIO;
872 goto out;
873 }
874
875 if (uio->uio_resid == 0) {
876 error = 0;
877 goto out;
878 }
879
880 error = bpf_movein(ifp, uio, (int)bp->bif_dlt, ifp->if_mtu, &m,
881 (struct sockaddr *) &dst, &d->bd_wfilter);
882 if (error)
883 goto out;
884
885 if (m->m_pkthdr.len > ifp->if_mtu) {
886 m_freem(m);
887 error = EMSGSIZE;
888 goto out;
889 }
890
891 /*
892 * If writing to a loopback interface, the address family has
893 * already been specially computed in bpf_movein(), so don't
894 * clobber it, or the loopback will reject it in looutput().
895 */
896 if (d->bd_hdrcmplt && ifp->if_type != IFT_LOOP)
897 dst.ss_family = pseudo_AF_HDRCMPLT;
898
899 if (d->bd_feedback) {
900 mc = m_dup(m, 0, M_COPYALL, M_NOWAIT);
901 if (mc != NULL)
902 m_set_rcvif(mc, ifp);
903 /* Set M_PROMISC for outgoing packets to be discarded. */
904 if (1 /*d->bd_direction == BPF_D_INOUT*/)
905 m->m_flags |= M_PROMISC;
906 } else
907 mc = NULL;
908
909 error = if_output_lock(ifp, ifp, m, (struct sockaddr *) &dst, NULL);
910
911 if (mc != NULL) {
912 if (error == 0) {
913 int s = splsoftnet();
914 KERNEL_LOCK_UNLESS_IFP_MPSAFE(ifp);
915 ifp->_if_input(ifp, mc);
916 KERNEL_UNLOCK_UNLESS_IFP_MPSAFE(ifp);
917 splx(s);
918 } else
919 m_freem(mc);
920 }
921 /*
922 * The driver frees the mbuf.
923 */
924 out:
925 bpf_if_release(bp, &psref);
926 out_bindx:
927 curlwp_bindx(bound);
928 return error;
929 }
930
931 /*
932 * Reset a descriptor by flushing its packet buffer and clearing the
933 * receive and drop counts.
934 */
935 static void
reset_d(struct bpf_d * d)936 reset_d(struct bpf_d *d)
937 {
938
939 KASSERT(mutex_owned(d->bd_mtx));
940
941 mutex_enter(d->bd_buf_mtx);
942 if (d->bd_hbuf) {
943 /* Free the hold buffer. */
944 d->bd_fbuf = d->bd_hbuf;
945 d->bd_hbuf = NULL;
946 }
947 d->bd_slen = 0;
948 d->bd_hlen = 0;
949 d->bd_rcount = 0;
950 d->bd_dcount = 0;
951 d->bd_ccount = 0;
952 mutex_exit(d->bd_buf_mtx);
953 }
954
955 /*
956 * FIONREAD Check for read packet available.
957 * BIOCGBLEN Get buffer len [for read()].
958 * BIOCSETF Set ethernet read filter.
959 * BIOCFLUSH Flush read packet buffer.
960 * BIOCPROMISC Put interface into promiscuous mode.
961 * BIOCGDLT Get link layer type.
962 * BIOCGETIF Get interface name.
963 * BIOCSETIF Set interface.
964 * BIOCSRTIMEOUT Set read timeout.
965 * BIOCGRTIMEOUT Get read timeout.
966 * BIOCGSTATS Get packet stats.
967 * BIOCIMMEDIATE Set immediate mode.
968 * BIOCVERSION Get filter language version.
969 * BIOCGHDRCMPLT Get "header already complete" flag.
970 * BIOCSHDRCMPLT Set "header already complete" flag.
971 * BIOCSFEEDBACK Set packet feedback mode.
972 * BIOCGFEEDBACK Get packet feedback mode.
973 * BIOCGDIRECTION Get packet direction flag
974 * BIOCSDIRECTION Set packet direction flag
975 */
976 /* ARGSUSED */
977 static int
bpf_ioctl(struct file * fp,u_long cmd,void * addr)978 bpf_ioctl(struct file *fp, u_long cmd, void *addr)
979 {
980 struct bpf_d *d = fp->f_bpf;
981 int error = 0;
982
983 /*
984 * Refresh the PID associated with this bpf file.
985 */
986 d->bd_pid = curproc->p_pid;
987 #ifdef _LP64
988 if (curproc->p_flag & PK_32)
989 d->bd_compat32 = 1;
990 else
991 d->bd_compat32 = 0;
992 #endif
993
994 mutex_enter(d->bd_buf_mtx);
995 if (d->bd_state == BPF_WAITING)
996 callout_halt(&d->bd_callout, d->bd_buf_mtx);
997 d->bd_state = BPF_IDLE;
998 mutex_exit(d->bd_buf_mtx);
999
1000 if (d->bd_locked) {
1001 switch (cmd) {
1002 case BIOCGBLEN: /* FALLTHROUGH */
1003 case BIOCFLUSH: /* FALLTHROUGH */
1004 case BIOCGDLT: /* FALLTHROUGH */
1005 case BIOCGDLTLIST: /* FALLTHROUGH */
1006 case BIOCGETIF: /* FALLTHROUGH */
1007 case BIOCGRTIMEOUT: /* FALLTHROUGH */
1008 case BIOCGSTATS: /* FALLTHROUGH */
1009 case BIOCVERSION: /* FALLTHROUGH */
1010 case BIOCGHDRCMPLT: /* FALLTHROUGH */
1011 case FIONREAD: /* FALLTHROUGH */
1012 case BIOCLOCK: /* FALLTHROUGH */
1013 case BIOCSRTIMEOUT: /* FALLTHROUGH */
1014 case BIOCIMMEDIATE: /* FALLTHROUGH */
1015 case TIOCGPGRP:
1016 break;
1017 default:
1018 return EPERM;
1019 }
1020 }
1021
1022 switch (cmd) {
1023
1024 default:
1025 error = EINVAL;
1026 break;
1027
1028 /*
1029 * Check for read packet available.
1030 */
1031 case FIONREAD: {
1032 int n;
1033
1034 mutex_enter(d->bd_buf_mtx);
1035 n = d->bd_slen;
1036 if (d->bd_hbuf)
1037 n += d->bd_hlen;
1038 mutex_exit(d->bd_buf_mtx);
1039
1040 *(int *)addr = n;
1041 break;
1042 }
1043
1044 /*
1045 * Get buffer len [for read()].
1046 */
1047 case BIOCGBLEN:
1048 *(u_int *)addr = d->bd_bufsize;
1049 break;
1050
1051 /*
1052 * Set buffer length.
1053 */
1054 case BIOCSBLEN:
1055 /*
1056 * Forbid to change the buffer length if buffers are already
1057 * allocated.
1058 */
1059 mutex_enter(d->bd_mtx);
1060 mutex_enter(d->bd_buf_mtx);
1061 if (d->bd_bif != NULL || d->bd_sbuf != NULL)
1062 error = EINVAL;
1063 else {
1064 u_int size = *(u_int *)addr;
1065
1066 if (size > bpf_maxbufsize)
1067 *(u_int *)addr = size = bpf_maxbufsize;
1068 else if (size < BPF_MINBUFSIZE)
1069 *(u_int *)addr = size = BPF_MINBUFSIZE;
1070 d->bd_bufsize = size;
1071 }
1072 mutex_exit(d->bd_buf_mtx);
1073 mutex_exit(d->bd_mtx);
1074 break;
1075
1076 /*
1077 * Set link layer read filter.
1078 */
1079 case BIOCSETF: /* FALLTHROUGH */
1080 case BIOCSETWF:
1081 error = bpf_setf(d, addr, cmd);
1082 break;
1083
1084 case BIOCLOCK:
1085 d->bd_locked = 1;
1086 break;
1087
1088 /*
1089 * Flush read packet buffer.
1090 */
1091 case BIOCFLUSH:
1092 mutex_enter(d->bd_mtx);
1093 reset_d(d);
1094 mutex_exit(d->bd_mtx);
1095 break;
1096
1097 /*
1098 * Put interface into promiscuous mode.
1099 */
1100 case BIOCPROMISC:
1101 mutex_enter(d->bd_mtx);
1102 if (d->bd_bif == NULL) {
1103 mutex_exit(d->bd_mtx);
1104 /*
1105 * No interface attached yet.
1106 */
1107 error = EINVAL;
1108 break;
1109 }
1110 if (d->bd_promisc == 0) {
1111 KERNEL_LOCK_UNLESS_NET_MPSAFE();
1112 error = ifpromisc(d->bd_bif->bif_ifp, 1);
1113 KERNEL_UNLOCK_UNLESS_NET_MPSAFE();
1114 if (error == 0)
1115 d->bd_promisc = 1;
1116 }
1117 mutex_exit(d->bd_mtx);
1118 break;
1119
1120 /*
1121 * Get device parameters.
1122 */
1123 case BIOCGDLT:
1124 mutex_enter(d->bd_mtx);
1125 if (d->bd_bif == NULL)
1126 error = EINVAL;
1127 else
1128 *(u_int *)addr = d->bd_bif->bif_dlt;
1129 mutex_exit(d->bd_mtx);
1130 break;
1131
1132 /*
1133 * Get a list of supported device parameters.
1134 */
1135 case BIOCGDLTLIST:
1136 mutex_enter(d->bd_mtx);
1137 if (d->bd_bif == NULL)
1138 error = EINVAL;
1139 else
1140 error = bpf_getdltlist(d, addr);
1141 mutex_exit(d->bd_mtx);
1142 break;
1143
1144 /*
1145 * Set device parameters.
1146 */
1147 case BIOCSDLT:
1148 mutex_enter(&bpf_mtx);
1149 mutex_enter(d->bd_mtx);
1150 if (d->bd_bif == NULL)
1151 error = EINVAL;
1152 else
1153 error = bpf_setdlt(d, *(u_int *)addr);
1154 mutex_exit(d->bd_mtx);
1155 mutex_exit(&bpf_mtx);
1156 break;
1157
1158 /*
1159 * Set interface name.
1160 */
1161 #ifdef OBIOCGETIF
1162 case OBIOCGETIF:
1163 #endif
1164 case BIOCGETIF:
1165 mutex_enter(d->bd_mtx);
1166 if (d->bd_bif == NULL)
1167 error = EINVAL;
1168 else
1169 bpf_ifname(d->bd_bif->bif_ifp, addr);
1170 mutex_exit(d->bd_mtx);
1171 break;
1172
1173 /*
1174 * Set interface.
1175 */
1176 #ifdef OBIOCSETIF
1177 case OBIOCSETIF:
1178 #endif
1179 case BIOCSETIF:
1180 mutex_enter(&bpf_mtx);
1181 error = bpf_setif(d, addr);
1182 mutex_exit(&bpf_mtx);
1183 break;
1184
1185 /*
1186 * Set read timeout.
1187 */
1188 case BIOCSRTIMEOUT: {
1189 struct timeval *tv = addr;
1190
1191 /* Compute number of ticks. */
1192 if (tv->tv_sec < 0 ||
1193 tv->tv_usec < 0 || tv->tv_usec >= 1000000) {
1194 error = EINVAL;
1195 break;
1196 } else if (tv->tv_sec > INT_MAX/hz - 1) {
1197 d->bd_rtout = INT_MAX;
1198 } else {
1199 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick;
1200 }
1201 if ((d->bd_rtout == 0) && (tv->tv_usec != 0))
1202 d->bd_rtout = 1;
1203 break;
1204 }
1205
1206 #ifdef BIOCGORTIMEOUT
1207 /*
1208 * Get read timeout.
1209 */
1210 case BIOCGORTIMEOUT: {
1211 struct timeval50 *tv = addr;
1212
1213 tv->tv_sec = d->bd_rtout / hz;
1214 tv->tv_usec = (d->bd_rtout % hz) * tick;
1215 break;
1216 }
1217 #endif
1218
1219 #ifdef BIOCSORTIMEOUT
1220 /*
1221 * Set read timeout.
1222 */
1223 case BIOCSORTIMEOUT: {
1224 struct timeval50 *tv = addr;
1225
1226 /* Compute number of ticks. */
1227 if (tv->tv_sec < 0 ||
1228 tv->tv_usec < 0 || tv->tv_usec >= 1000000) {
1229 error = EINVAL;
1230 break;
1231 } else if (tv->tv_sec > INT_MAX/hz - 1) {
1232 d->bd_rtout = INT_MAX;
1233 } else {
1234 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick;
1235 }
1236 if ((d->bd_rtout == 0) && (tv->tv_usec != 0))
1237 d->bd_rtout = 1;
1238 break;
1239 }
1240 #endif
1241
1242 /*
1243 * Get read timeout.
1244 */
1245 case BIOCGRTIMEOUT: {
1246 struct timeval *tv = addr;
1247
1248 tv->tv_sec = d->bd_rtout / hz;
1249 tv->tv_usec = (d->bd_rtout % hz) * tick;
1250 break;
1251 }
1252 /*
1253 * Get packet stats.
1254 */
1255 case BIOCGSTATS: {
1256 struct bpf_stat *bs = addr;
1257
1258 bs->bs_recv = d->bd_rcount;
1259 bs->bs_drop = d->bd_dcount;
1260 bs->bs_capt = d->bd_ccount;
1261 break;
1262 }
1263
1264 case BIOCGSTATS_30: {
1265 struct bpf_stat30 *bs = addr;
1266
1267 bs->bs_recv = d->bd_rcount;
1268 bs->bs_drop = d->bd_dcount;
1269 break;
1270 }
1271
1272 /*
1273 * Set immediate mode.
1274 */
1275 case BIOCIMMEDIATE:
1276 d->bd_immediate = *(u_int *)addr;
1277 break;
1278
1279 case BIOCVERSION: {
1280 struct bpf_version *bv = addr;
1281
1282 bv->bv_major = BPF_MAJOR_VERSION;
1283 bv->bv_minor = BPF_MINOR_VERSION;
1284 break;
1285 }
1286
1287 case BIOCGHDRCMPLT: /* get "header already complete" flag */
1288 *(u_int *)addr = d->bd_hdrcmplt;
1289 break;
1290
1291 case BIOCSHDRCMPLT: /* set "header already complete" flag */
1292 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0;
1293 break;
1294
1295 /*
1296 * Get packet direction flag
1297 */
1298 case BIOCGDIRECTION:
1299 *(u_int *)addr = d->bd_direction;
1300 break;
1301
1302 /*
1303 * Set packet direction flag
1304 */
1305 case BIOCSDIRECTION: {
1306 u_int direction;
1307
1308 direction = *(u_int *)addr;
1309 switch (direction) {
1310 case BPF_D_IN:
1311 case BPF_D_INOUT:
1312 case BPF_D_OUT:
1313 d->bd_direction = direction;
1314 break;
1315 default:
1316 error = EINVAL;
1317 }
1318 }
1319 break;
1320
1321 /*
1322 * Set "feed packets from bpf back to input" mode
1323 */
1324 case BIOCSFEEDBACK:
1325 d->bd_feedback = *(u_int *)addr;
1326 break;
1327
1328 /*
1329 * Get "feed packets from bpf back to input" mode
1330 */
1331 case BIOCGFEEDBACK:
1332 *(u_int *)addr = d->bd_feedback;
1333 break;
1334
1335 case FIONBIO: /* Non-blocking I/O */
1336 /*
1337 * No need to do anything special as we use IO_NDELAY in
1338 * bpfread() as an indication of whether or not to block
1339 * the read.
1340 */
1341 break;
1342
1343 case FIOASYNC: /* Send signal on receive packets */
1344 mutex_enter(d->bd_mtx);
1345 d->bd_async = *(int *)addr;
1346 mutex_exit(d->bd_mtx);
1347 break;
1348
1349 case TIOCSPGRP: /* Process or group to send signals to */
1350 case FIOSETOWN:
1351 error = fsetown(&d->bd_pgid, cmd, addr);
1352 break;
1353
1354 case TIOCGPGRP:
1355 case FIOGETOWN:
1356 error = fgetown(d->bd_pgid, cmd, addr);
1357 break;
1358 }
1359 return (error);
1360 }
1361
1362 /*
1363 * Set d's packet filter program to fp. If this file already has a filter,
1364 * free it and replace it. Returns EINVAL for bogus requests.
1365 */
1366 static int
bpf_setf(struct bpf_d * d,struct bpf_program * fp,u_long cmd)1367 bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd)
1368 {
1369 struct bpf_insn *fcode;
1370 bpfjit_func_t jcode;
1371 size_t flen, size = 0;
1372 struct bpf_filter *oldf, *newf, **storef;
1373
1374 jcode = NULL;
1375 flen = fp->bf_len;
1376
1377 if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) {
1378 return EINVAL;
1379 }
1380
1381 if (flen) {
1382 /*
1383 * Allocate the buffer, copy the byte-code from
1384 * userspace and validate it.
1385 */
1386 size = flen * sizeof(*fp->bf_insns);
1387 fcode = kmem_alloc(size, KM_SLEEP);
1388 if (copyin(fp->bf_insns, fcode, size) != 0 ||
1389 !bpf_validate(fcode, (int)flen)) {
1390 kmem_free(fcode, size);
1391 return EINVAL;
1392 }
1393 if (bpf_jit)
1394 jcode = bpf_jit_generate(NULL, fcode, flen);
1395 } else {
1396 fcode = NULL;
1397 }
1398
1399 newf = kmem_alloc(sizeof(*newf), KM_SLEEP);
1400 newf->bf_insn = fcode;
1401 newf->bf_size = size;
1402 newf->bf_jitcode = jcode;
1403 if (cmd == BIOCSETF)
1404 d->bd_jitcode = jcode; /* XXX just for kvm(3) users */
1405
1406 /* Need to hold bpf_mtx for pserialize_perform */
1407 mutex_enter(&bpf_mtx);
1408 mutex_enter(d->bd_mtx);
1409 if (cmd == BIOCSETWF) {
1410 oldf = d->bd_wfilter;
1411 storef = &d->bd_wfilter;
1412 } else {
1413 oldf = d->bd_rfilter;
1414 storef = &d->bd_rfilter;
1415 }
1416 atomic_store_release(storef, newf);
1417 reset_d(d);
1418 pserialize_perform(bpf_psz);
1419 mutex_exit(d->bd_mtx);
1420 mutex_exit(&bpf_mtx);
1421
1422 if (oldf != NULL)
1423 bpf_free_filter(oldf);
1424
1425 return 0;
1426 }
1427
1428 /*
1429 * Detach a file from its current interface (if attached at all) and attach
1430 * to the interface indicated by the name stored in ifr.
1431 * Return an errno or 0.
1432 */
1433 static int
bpf_setif(struct bpf_d * d,struct ifreq * ifr)1434 bpf_setif(struct bpf_d *d, struct ifreq *ifr)
1435 {
1436 struct bpf_if *bp;
1437 char *cp;
1438 int unit_seen, i, error;
1439
1440 KASSERT(mutex_owned(&bpf_mtx));
1441 /*
1442 * Make sure the provided name has a unit number, and default
1443 * it to '0' if not specified.
1444 * XXX This is ugly ... do this differently?
1445 */
1446 unit_seen = 0;
1447 cp = ifr->ifr_name;
1448 cp[sizeof(ifr->ifr_name) - 1] = '\0'; /* sanity */
1449 while (*cp++)
1450 if (*cp >= '0' && *cp <= '9')
1451 unit_seen = 1;
1452 if (!unit_seen) {
1453 /* Make sure to leave room for the '\0'. */
1454 for (i = 0; i < (IFNAMSIZ - 1); ++i) {
1455 if ((ifr->ifr_name[i] >= 'a' &&
1456 ifr->ifr_name[i] <= 'z') ||
1457 (ifr->ifr_name[i] >= 'A' &&
1458 ifr->ifr_name[i] <= 'Z'))
1459 continue;
1460 ifr->ifr_name[i] = '0';
1461 }
1462 }
1463
1464 /*
1465 * Look through attached interfaces for the named one.
1466 */
1467 BPF_IFLIST_WRITER_FOREACH(bp) {
1468 struct ifnet *ifp = bp->bif_ifp;
1469
1470 if (ifp == NULL ||
1471 strcmp(ifp->if_xname, ifr->ifr_name) != 0)
1472 continue;
1473 /* skip additional entry */
1474 if (bp->bif_driverp != &ifp->if_bpf)
1475 continue;
1476 /*
1477 * We found the requested interface.
1478 * Allocate the packet buffers if we need to.
1479 * If we're already attached to requested interface,
1480 * just flush the buffer.
1481 */
1482 /*
1483 * bpf_allocbufs is called only here. bpf_mtx ensures that
1484 * no race condition happen on d->bd_sbuf.
1485 */
1486 if (d->bd_sbuf == NULL) {
1487 error = bpf_allocbufs(d);
1488 if (error != 0)
1489 return (error);
1490 }
1491 mutex_enter(d->bd_mtx);
1492 if (bp != d->bd_bif) {
1493 if (d->bd_bif) {
1494 /*
1495 * Detach if attached to something else.
1496 */
1497 bpf_detachd(d);
1498 BPFIF_DLIST_ENTRY_INIT(d);
1499 }
1500
1501 bpf_attachd(d, bp);
1502 }
1503 reset_d(d);
1504 mutex_exit(d->bd_mtx);
1505 return (0);
1506 }
1507 /* Not found. */
1508 return (ENXIO);
1509 }
1510
1511 /*
1512 * Copy the interface name to the ifreq.
1513 */
1514 static void
bpf_ifname(struct ifnet * ifp,struct ifreq * ifr)1515 bpf_ifname(struct ifnet *ifp, struct ifreq *ifr)
1516 {
1517 memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ);
1518 }
1519
1520 static int
bpf_stat(struct file * fp,struct stat * st)1521 bpf_stat(struct file *fp, struct stat *st)
1522 {
1523 struct bpf_d *d = fp->f_bpf;
1524
1525 (void)memset(st, 0, sizeof(*st));
1526 mutex_enter(d->bd_mtx);
1527 st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid);
1528 st->st_atimespec = d->bd_atime;
1529 st->st_mtimespec = d->bd_mtime;
1530 st->st_ctimespec = st->st_birthtimespec = d->bd_btime;
1531 st->st_uid = kauth_cred_geteuid(fp->f_cred);
1532 st->st_gid = kauth_cred_getegid(fp->f_cred);
1533 st->st_mode = S_IFCHR;
1534 mutex_exit(d->bd_mtx);
1535 return 0;
1536 }
1537
1538 /*
1539 * Support for poll() system call
1540 *
1541 * Return true iff the specific operation will not block indefinitely - with
1542 * the assumption that it is safe to positively acknowledge a request for the
1543 * ability to write to the BPF device.
1544 * Otherwise, return false but make a note that a selnotify() must be done.
1545 */
1546 static int
bpf_poll(struct file * fp,int events)1547 bpf_poll(struct file *fp, int events)
1548 {
1549 struct bpf_d *d = fp->f_bpf;
1550 int revents;
1551
1552 /*
1553 * Refresh the PID associated with this bpf file.
1554 */
1555 mutex_enter(&bpf_mtx);
1556 d->bd_pid = curproc->p_pid;
1557
1558 revents = events & (POLLOUT | POLLWRNORM);
1559 if (events & (POLLIN | POLLRDNORM)) {
1560 /*
1561 * An imitation of the FIONREAD ioctl code.
1562 */
1563 mutex_enter(d->bd_mtx);
1564 mutex_enter(d->bd_buf_mtx);
1565 if (d->bd_hlen != 0 ||
1566 ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) &&
1567 d->bd_slen != 0)) {
1568 revents |= events & (POLLIN | POLLRDNORM);
1569 } else {
1570 selrecord(curlwp, &d->bd_sel);
1571 /* Start the read timeout if necessary */
1572 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
1573 callout_reset(&d->bd_callout, d->bd_rtout,
1574 bpf_timed_out, d);
1575 d->bd_state = BPF_WAITING;
1576 }
1577 }
1578 mutex_exit(d->bd_buf_mtx);
1579 mutex_exit(d->bd_mtx);
1580 }
1581
1582 mutex_exit(&bpf_mtx);
1583 return (revents);
1584 }
1585
1586 static void
filt_bpfrdetach(struct knote * kn)1587 filt_bpfrdetach(struct knote *kn)
1588 {
1589 struct bpf_d *d = kn->kn_hook;
1590
1591 mutex_enter(d->bd_buf_mtx);
1592 selremove_knote(&d->bd_sel, kn);
1593 mutex_exit(d->bd_buf_mtx);
1594 }
1595
1596 static int
filt_bpfread(struct knote * kn,long hint)1597 filt_bpfread(struct knote *kn, long hint)
1598 {
1599 struct bpf_d *d = kn->kn_hook;
1600 int rv;
1601
1602 /*
1603 * Refresh the PID associated with this bpf file.
1604 */
1605 d->bd_pid = curproc->p_pid;
1606
1607 if (hint & NOTE_SUBMIT)
1608 KASSERT(mutex_owned(d->bd_buf_mtx));
1609 else
1610 mutex_enter(d->bd_buf_mtx);
1611 kn->kn_data = d->bd_hlen;
1612 if (d->bd_immediate)
1613 kn->kn_data += d->bd_slen;
1614 rv = (kn->kn_data > 0);
1615 if (hint & NOTE_SUBMIT)
1616 KASSERT(mutex_owned(d->bd_buf_mtx));
1617 else
1618 mutex_exit(d->bd_buf_mtx);
1619 return rv;
1620 }
1621
1622 static const struct filterops bpfread_filtops = {
1623 .f_flags = FILTEROP_ISFD | FILTEROP_MPSAFE,
1624 .f_attach = NULL,
1625 .f_detach = filt_bpfrdetach,
1626 .f_event = filt_bpfread,
1627 };
1628
1629 static int
bpf_kqfilter(struct file * fp,struct knote * kn)1630 bpf_kqfilter(struct file *fp, struct knote *kn)
1631 {
1632 struct bpf_d *d = fp->f_bpf;
1633
1634 switch (kn->kn_filter) {
1635 case EVFILT_READ:
1636 kn->kn_fop = &bpfread_filtops;
1637 break;
1638
1639 default:
1640 return (EINVAL);
1641 }
1642
1643 kn->kn_hook = d;
1644
1645 mutex_enter(d->bd_buf_mtx);
1646 selrecord_knote(&d->bd_sel, kn);
1647 mutex_exit(d->bd_buf_mtx);
1648
1649 return (0);
1650 }
1651
1652 /*
1653 * Copy data from an mbuf chain into a buffer. This code is derived
1654 * from m_copydata in sys/uipc_mbuf.c.
1655 */
1656 static void *
bpf_mcpy(void * dst_arg,const void * src_arg,size_t len)1657 bpf_mcpy(void *dst_arg, const void *src_arg, size_t len)
1658 {
1659 const struct mbuf *m;
1660 u_int count;
1661 u_char *dst;
1662
1663 m = src_arg;
1664 dst = dst_arg;
1665 while (len > 0) {
1666 if (m == NULL)
1667 panic("bpf_mcpy");
1668 count = uimin(m->m_len, len);
1669 memcpy(dst, mtod(m, const void *), count);
1670 m = m->m_next;
1671 dst += count;
1672 len -= count;
1673 }
1674 return dst_arg;
1675 }
1676
1677 static inline u_int
bpf_xfilter(struct bpf_filter ** filter,void * pkt,u_int pktlen,u_int buflen)1678 bpf_xfilter(struct bpf_filter **filter, void *pkt, u_int pktlen, u_int buflen)
1679 {
1680 struct bpf_filter *filt;
1681 uint32_t mem[BPF_MEMWORDS];
1682 bpf_args_t args = {
1683 .pkt = (const uint8_t *)pkt,
1684 .wirelen = pktlen,
1685 .buflen = buflen,
1686 .mem = mem,
1687 .arg = NULL
1688 };
1689 u_int slen;
1690
1691 filt = atomic_load_consume(filter);
1692 if (filt == NULL) /* No filter means accept all. */
1693 return (u_int)-1;
1694
1695 if (filt->bf_jitcode != NULL)
1696 slen = filt->bf_jitcode(NULL, &args);
1697 else
1698 slen = bpf_filter_ext(NULL, filt->bf_insn, &args);
1699 return slen;
1700 }
1701
1702 /*
1703 * Dispatch a packet to all the listeners on interface bp.
1704 *
1705 * pkt pointer to the packet, either a data buffer or an mbuf chain
1706 * buflen buffer length, if pkt is a data buffer
1707 * cpfn a function that can copy pkt into the listener's buffer
1708 * pktlen length of the packet
1709 * direction BPF_D_IN or BPF_D_OUT
1710 */
1711 static inline void
bpf_deliver(struct bpf_if * bp,void * (* cpfn)(void *,const void *,size_t),void * pkt,u_int pktlen,u_int buflen,const u_int direction)1712 bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t),
1713 void *pkt, u_int pktlen, u_int buflen, const u_int direction)
1714 {
1715 bool gottime = false;
1716 struct timespec ts;
1717 struct bpf_d *d;
1718 int s;
1719 u_int slen;
1720
1721 KASSERT(!cpu_intr_p());
1722
1723 /*
1724 * Note that the IPL does not have to be raised at this point.
1725 * The only problem that could arise here is that if two different
1726 * interfaces shared any data. This is not the case.
1727 */
1728 s = pserialize_read_enter();
1729 BPFIF_DLIST_READER_FOREACH(d, bp) {
1730 if (direction == BPF_D_IN) {
1731 if (d->bd_direction == BPF_D_OUT)
1732 continue;
1733 } else { /* BPF_D_OUT */
1734 if (d->bd_direction == BPF_D_IN)
1735 continue;
1736 }
1737
1738 atomic_inc_ulong(&d->bd_rcount);
1739 BPF_STATINC(recv);
1740
1741 slen = bpf_xfilter(&d->bd_rfilter, pkt, pktlen, buflen);
1742 if (slen == 0)
1743 continue;
1744
1745 if (!gottime) {
1746 gottime = true;
1747 nanotime(&ts);
1748 }
1749 /* Assume catchpacket doesn't sleep */
1750 catchpacket(d, pkt, pktlen, slen, cpfn, &ts);
1751 }
1752 pserialize_read_exit(s);
1753 }
1754
1755 /*
1756 * Incoming linkage from device drivers, when the head of the packet is in
1757 * a buffer, and the tail is in an mbuf chain.
1758 */
1759 static void
_bpf_mtap2(struct bpf_if * bp,void * data,u_int dlen,struct mbuf * m,u_int direction)1760 _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m,
1761 u_int direction)
1762 {
1763 u_int pktlen;
1764 struct mbuf mb;
1765
1766 /* Skip outgoing duplicate packets. */
1767 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) {
1768 m->m_flags &= ~M_PROMISC;
1769 return;
1770 }
1771
1772 pktlen = m_length(m) + dlen;
1773
1774 /*
1775 * Craft on-stack mbuf suitable for passing to bpf_filter.
1776 * Note that we cut corners here; we only set up what's
1777 * absolutely needed--this mbuf should never go anywhere else.
1778 */
1779 (void)memset(&mb, 0, sizeof(mb));
1780 mb.m_type = MT_DATA;
1781 mb.m_next = m;
1782 mb.m_data = data;
1783 mb.m_len = dlen;
1784
1785 bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, direction);
1786 }
1787
1788 /*
1789 * Incoming linkage from device drivers, when packet is in an mbuf chain.
1790 */
1791 static void
_bpf_mtap(struct bpf_if * bp,struct mbuf * m,u_int direction)1792 _bpf_mtap(struct bpf_if *bp, struct mbuf *m, u_int direction)
1793 {
1794 void *(*cpfn)(void *, const void *, size_t);
1795 u_int pktlen, buflen;
1796 void *marg;
1797
1798 /* Skip outgoing duplicate packets. */
1799 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) {
1800 m->m_flags &= ~M_PROMISC;
1801 return;
1802 }
1803
1804 pktlen = m_length(m);
1805
1806 /* Skip zero-sized packets. */
1807 if (__predict_false(pktlen == 0)) {
1808 return;
1809 }
1810
1811 if (pktlen == m->m_len) {
1812 cpfn = (void *)memcpy;
1813 marg = mtod(m, void *);
1814 buflen = pktlen;
1815 KASSERT(buflen != 0);
1816 } else {
1817 cpfn = bpf_mcpy;
1818 marg = m;
1819 buflen = 0;
1820 }
1821
1822 bpf_deliver(bp, cpfn, marg, pktlen, buflen, direction);
1823 }
1824
1825 /*
1826 * We need to prepend the address family as
1827 * a four byte field. Cons up a dummy header
1828 * to pacify bpf. This is safe because bpf
1829 * will only read from the mbuf (i.e., it won't
1830 * try to free it or keep a pointer a to it).
1831 */
1832 static void
_bpf_mtap_af(struct bpf_if * bp,uint32_t af,struct mbuf * m,u_int direction)1833 _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m, u_int direction)
1834 {
1835 struct mbuf m0;
1836
1837 m0.m_type = MT_DATA;
1838 m0.m_flags = 0;
1839 m0.m_next = m;
1840 m0.m_nextpkt = NULL;
1841 m0.m_owner = NULL;
1842 m0.m_len = 4;
1843 m0.m_data = (char *)⁡
1844
1845 _bpf_mtap(bp, &m0, direction);
1846 }
1847
1848 /*
1849 * Put the SLIP pseudo-"link header" in place.
1850 * Note this M_PREPEND() should never fail,
1851 * since we know we always have enough space
1852 * in the input buffer.
1853 */
1854 static void
_bpf_mtap_sl_in(struct bpf_if * bp,u_char * chdr,struct mbuf ** m)1855 _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m)
1856 {
1857 u_char *hp;
1858
1859 M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT);
1860 if (*m == NULL)
1861 return;
1862
1863 hp = mtod(*m, u_char *);
1864 hp[SLX_DIR] = SLIPDIR_IN;
1865 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN);
1866
1867 _bpf_mtap(bp, *m, BPF_D_IN);
1868
1869 m_adj(*m, SLIP_HDRLEN);
1870 }
1871
1872 /*
1873 * Put the SLIP pseudo-"link header" in
1874 * place. The compressed header is now
1875 * at the beginning of the mbuf.
1876 */
1877 static void
_bpf_mtap_sl_out(struct bpf_if * bp,u_char * chdr,struct mbuf * m)1878 _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m)
1879 {
1880 struct mbuf m0;
1881 u_char *hp;
1882
1883 m0.m_type = MT_DATA;
1884 m0.m_flags = 0;
1885 m0.m_next = m;
1886 m0.m_nextpkt = NULL;
1887 m0.m_owner = NULL;
1888 m0.m_data = m0.m_dat;
1889 m0.m_len = SLIP_HDRLEN;
1890
1891 hp = mtod(&m0, u_char *);
1892
1893 hp[SLX_DIR] = SLIPDIR_OUT;
1894 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN);
1895
1896 _bpf_mtap(bp, &m0, BPF_D_OUT);
1897 m_freem(m);
1898 }
1899
1900 static struct mbuf *
bpf_mbuf_enqueue(struct bpf_if * bp,struct mbuf * m)1901 bpf_mbuf_enqueue(struct bpf_if *bp, struct mbuf *m)
1902 {
1903 struct mbuf *dup;
1904
1905 dup = m_dup(m, 0, M_COPYALL, M_NOWAIT);
1906 if (dup == NULL)
1907 return NULL;
1908
1909 if (bp->bif_mbuf_tail != NULL) {
1910 bp->bif_mbuf_tail->m_nextpkt = dup;
1911 } else {
1912 bp->bif_mbuf_head = dup;
1913 }
1914 bp->bif_mbuf_tail = dup;
1915 #ifdef BPF_MTAP_SOFTINT_DEBUG
1916 log(LOG_DEBUG, "%s: enqueued mbuf=%p to %s\n",
1917 __func__, dup, bp->bif_ifp->if_xname);
1918 #endif
1919
1920 return dup;
1921 }
1922
1923 static struct mbuf *
bpf_mbuf_dequeue(struct bpf_if * bp)1924 bpf_mbuf_dequeue(struct bpf_if *bp)
1925 {
1926 struct mbuf *m;
1927 int s;
1928
1929 /* XXX NOMPSAFE: assumed running on one CPU */
1930 s = splnet();
1931 m = bp->bif_mbuf_head;
1932 if (m != NULL) {
1933 bp->bif_mbuf_head = m->m_nextpkt;
1934 m->m_nextpkt = NULL;
1935
1936 if (bp->bif_mbuf_head == NULL)
1937 bp->bif_mbuf_tail = NULL;
1938 #ifdef BPF_MTAP_SOFTINT_DEBUG
1939 log(LOG_DEBUG, "%s: dequeued mbuf=%p from %s\n",
1940 __func__, m, bp->bif_ifp->if_xname);
1941 #endif
1942 }
1943 splx(s);
1944
1945 return m;
1946 }
1947
1948 static void
bpf_mtap_si(void * arg)1949 bpf_mtap_si(void *arg)
1950 {
1951 struct bpf_if *bp = arg;
1952 struct mbuf *m;
1953
1954 while ((m = bpf_mbuf_dequeue(bp)) != NULL) {
1955 #ifdef BPF_MTAP_SOFTINT_DEBUG
1956 log(LOG_DEBUG, "%s: tapping mbuf=%p on %s\n",
1957 __func__, m, bp->bif_ifp->if_xname);
1958 #endif
1959 bpf_ops->bpf_mtap(bp, m, BPF_D_IN);
1960 m_freem(m);
1961 }
1962 }
1963
1964 static void
_bpf_mtap_softint(struct ifnet * ifp,struct mbuf * m)1965 _bpf_mtap_softint(struct ifnet *ifp, struct mbuf *m)
1966 {
1967 struct bpf_if *bp = ifp->if_bpf;
1968 struct mbuf *dup;
1969
1970 KASSERT(cpu_intr_p());
1971
1972 /* To avoid extra invocations of the softint */
1973 if (BPFIF_DLIST_READER_EMPTY(bp))
1974 return;
1975 KASSERT(bp->bif_si != NULL);
1976
1977 dup = bpf_mbuf_enqueue(bp, m);
1978 if (dup != NULL)
1979 softint_schedule(bp->bif_si);
1980 }
1981
1982 static int
bpf_hdrlen(struct bpf_d * d)1983 bpf_hdrlen(struct bpf_d *d)
1984 {
1985 int hdrlen = d->bd_bif->bif_hdrlen;
1986 /*
1987 * Compute the length of the bpf header. This is not necessarily
1988 * equal to SIZEOF_BPF_HDR because we want to insert spacing such
1989 * that the network layer header begins on a longword boundary (for
1990 * performance reasons and to alleviate alignment restrictions).
1991 */
1992 #ifdef _LP64
1993 if (d->bd_compat32)
1994 return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen);
1995 else
1996 #endif
1997 return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen);
1998 }
1999
2000 /*
2001 * Move the packet data from interface memory (pkt) into the
2002 * store buffer. Call the wakeup functions if it's time to wake up
2003 * a listener (buffer full), "cpfn" is the routine called to do the
2004 * actual data transfer. memcpy is passed in to copy contiguous chunks,
2005 * while bpf_mcpy is passed in to copy mbuf chains. In the latter case,
2006 * pkt is really an mbuf.
2007 */
2008 static void
catchpacket(struct bpf_d * d,u_char * pkt,u_int pktlen,u_int snaplen,void * (* cpfn)(void *,const void *,size_t),struct timespec * ts)2009 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen,
2010 void *(*cpfn)(void *, const void *, size_t), struct timespec *ts)
2011 {
2012 char *h;
2013 int totlen, curlen, caplen;
2014 int hdrlen = bpf_hdrlen(d);
2015 int do_wakeup = 0;
2016
2017 atomic_inc_ulong(&d->bd_ccount);
2018 BPF_STATINC(capt);
2019 /*
2020 * Figure out how many bytes to move. If the packet is
2021 * greater or equal to the snapshot length, transfer that
2022 * much. Otherwise, transfer the whole packet (unless
2023 * we hit the buffer size limit).
2024 */
2025 totlen = hdrlen + uimin(snaplen, pktlen);
2026 if (totlen > d->bd_bufsize)
2027 totlen = d->bd_bufsize;
2028 /*
2029 * If we adjusted totlen to fit the bufsize, it could be that
2030 * totlen is smaller than hdrlen because of the link layer header.
2031 */
2032 caplen = totlen - hdrlen;
2033 if (caplen < 0)
2034 caplen = 0;
2035
2036 mutex_enter(d->bd_buf_mtx);
2037 /*
2038 * Round up the end of the previous packet to the next longword.
2039 */
2040 #ifdef _LP64
2041 if (d->bd_compat32)
2042 curlen = BPF_WORDALIGN32(d->bd_slen);
2043 else
2044 #endif
2045 curlen = BPF_WORDALIGN(d->bd_slen);
2046 if (curlen + totlen > d->bd_bufsize) {
2047 /*
2048 * This packet will overflow the storage buffer.
2049 * Rotate the buffers if we can, then wakeup any
2050 * pending reads.
2051 */
2052 if (d->bd_fbuf == NULL) {
2053 mutex_exit(d->bd_buf_mtx);
2054 /*
2055 * We haven't completed the previous read yet,
2056 * so drop the packet.
2057 */
2058 atomic_inc_ulong(&d->bd_dcount);
2059 BPF_STATINC(drop);
2060 return;
2061 }
2062 ROTATE_BUFFERS(d);
2063 do_wakeup = 1;
2064 curlen = 0;
2065 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) {
2066 /*
2067 * Immediate mode is set, or the read timeout has
2068 * already expired during a select call. A packet
2069 * arrived, so the reader should be woken up.
2070 */
2071 do_wakeup = 1;
2072 }
2073
2074 /*
2075 * Append the bpf header.
2076 */
2077 h = (char *)d->bd_sbuf + curlen;
2078 #ifdef _LP64
2079 if (d->bd_compat32) {
2080 struct bpf_hdr32 *hp32;
2081
2082 hp32 = (struct bpf_hdr32 *)h;
2083 hp32->bh_tstamp.tv_sec = ts->tv_sec;
2084 hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000;
2085 hp32->bh_datalen = pktlen;
2086 hp32->bh_hdrlen = hdrlen;
2087 hp32->bh_caplen = caplen;
2088 } else
2089 #endif
2090 {
2091 struct bpf_hdr *hp;
2092
2093 hp = (struct bpf_hdr *)h;
2094 hp->bh_tstamp.tv_sec = ts->tv_sec;
2095 hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000;
2096 hp->bh_datalen = pktlen;
2097 hp->bh_hdrlen = hdrlen;
2098 hp->bh_caplen = caplen;
2099 }
2100
2101 /*
2102 * Copy the packet data into the store buffer and update its length.
2103 */
2104 (*cpfn)(h + hdrlen, pkt, caplen);
2105 d->bd_slen = curlen + totlen;
2106
2107 /*
2108 * Call bpf_wakeup after bd_slen has been updated so that kevent(2)
2109 * will cause filt_bpfread() to be called with it adjusted.
2110 */
2111 if (do_wakeup)
2112 bpf_wakeup(d);
2113
2114 mutex_exit(d->bd_buf_mtx);
2115 }
2116
2117 /*
2118 * Initialize all nonzero fields of a descriptor.
2119 */
2120 static int
bpf_allocbufs(struct bpf_d * d)2121 bpf_allocbufs(struct bpf_d *d)
2122 {
2123
2124 d->bd_fbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP);
2125 if (!d->bd_fbuf)
2126 return (ENOBUFS);
2127 d->bd_sbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP);
2128 if (!d->bd_sbuf) {
2129 kmem_free(d->bd_fbuf, d->bd_bufsize);
2130 return (ENOBUFS);
2131 }
2132 d->bd_slen = 0;
2133 d->bd_hlen = 0;
2134 return (0);
2135 }
2136
2137 static void
bpf_free_filter(struct bpf_filter * filter)2138 bpf_free_filter(struct bpf_filter *filter)
2139 {
2140
2141 KASSERT(filter != NULL);
2142
2143 if (filter->bf_insn != NULL)
2144 kmem_free(filter->bf_insn, filter->bf_size);
2145 if (filter->bf_jitcode != NULL)
2146 bpf_jit_freecode(filter->bf_jitcode);
2147 kmem_free(filter, sizeof(*filter));
2148 }
2149
2150 /*
2151 * Free buffers currently in use by a descriptor.
2152 * Called on close.
2153 */
2154 static void
bpf_freed(struct bpf_d * d)2155 bpf_freed(struct bpf_d *d)
2156 {
2157 /*
2158 * We don't need to lock out interrupts since this descriptor has
2159 * been detached from its interface and it yet hasn't been marked
2160 * free.
2161 */
2162 if (d->bd_sbuf != NULL) {
2163 kmem_free(d->bd_sbuf, d->bd_bufsize);
2164 if (d->bd_hbuf != NULL)
2165 kmem_free(d->bd_hbuf, d->bd_bufsize);
2166 if (d->bd_fbuf != NULL)
2167 kmem_free(d->bd_fbuf, d->bd_bufsize);
2168 }
2169 if (d->bd_rfilter != NULL) {
2170 bpf_free_filter(d->bd_rfilter);
2171 d->bd_rfilter = NULL;
2172 }
2173 if (d->bd_wfilter != NULL) {
2174 bpf_free_filter(d->bd_wfilter);
2175 d->bd_wfilter = NULL;
2176 }
2177 d->bd_jitcode = NULL;
2178 }
2179
2180 /*
2181 * Attach an interface to bpf. dlt is the link layer type;
2182 * hdrlen is the fixed size of the link header for the specified dlt
2183 * (variable length headers not yet supported).
2184 */
2185 static void
_bpfattach(struct ifnet * ifp,u_int dlt,u_int hdrlen,struct bpf_if ** driverp)2186 _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp)
2187 {
2188 struct bpf_if *bp;
2189
2190 bp = kmem_alloc(sizeof(*bp), KM_SLEEP);
2191
2192 mutex_enter(&bpf_mtx);
2193 bp->bif_driverp = driverp;
2194 bp->bif_ifp = ifp;
2195 bp->bif_dlt = dlt;
2196 bp->bif_si = NULL;
2197 BPF_IFLIST_ENTRY_INIT(bp);
2198 PSLIST_INIT(&bp->bif_dlist_head);
2199 psref_target_init(&bp->bif_psref, bpf_psref_class);
2200 SLIST_INIT(&bp->bif_trackers);
2201
2202 BPF_IFLIST_WRITER_INSERT_HEAD(bp);
2203
2204 *bp->bif_driverp = NULL;
2205
2206 bp->bif_hdrlen = hdrlen;
2207 mutex_exit(&bpf_mtx);
2208 #if 0
2209 printf("bpf: %s attached with dlt %x\n", ifp->if_xname, dlt);
2210 #endif
2211 }
2212
2213 static void
_bpf_mtap_softint_init(struct ifnet * ifp)2214 _bpf_mtap_softint_init(struct ifnet *ifp)
2215 {
2216 struct bpf_if *bp;
2217
2218 mutex_enter(&bpf_mtx);
2219 BPF_IFLIST_WRITER_FOREACH(bp) {
2220 if (bp->bif_ifp != ifp)
2221 continue;
2222
2223 bp->bif_mbuf_head = NULL;
2224 bp->bif_mbuf_tail = NULL;
2225 bp->bif_si = softint_establish(SOFTINT_NET, bpf_mtap_si, bp);
2226 if (bp->bif_si == NULL)
2227 panic("%s: softint_establish() failed", __func__);
2228 break;
2229 }
2230 mutex_exit(&bpf_mtx);
2231
2232 if (bp == NULL)
2233 panic("%s: no bpf_if found for %s", __func__, ifp->if_xname);
2234 }
2235
2236 /*
2237 * Remove an interface from bpf.
2238 */
2239 static void
_bpfdetach(struct ifnet * ifp)2240 _bpfdetach(struct ifnet *ifp)
2241 {
2242 struct bpf_if *bp;
2243 struct bpf_d *d;
2244 int s;
2245
2246 mutex_enter(&bpf_mtx);
2247 /* Nuke the vnodes for any open instances */
2248 again_d:
2249 BPF_DLIST_WRITER_FOREACH(d) {
2250 mutex_enter(d->bd_mtx);
2251 if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) {
2252 /*
2253 * Detach the descriptor from an interface now.
2254 * It will be free'ed later by close routine.
2255 */
2256 bpf_detachd(d);
2257 mutex_exit(d->bd_mtx);
2258 goto again_d;
2259 }
2260 mutex_exit(d->bd_mtx);
2261 }
2262
2263 again:
2264 BPF_IFLIST_WRITER_FOREACH(bp) {
2265 if (bp->bif_ifp == ifp) {
2266 BPF_IFLIST_WRITER_REMOVE(bp);
2267
2268 pserialize_perform(bpf_psz);
2269 psref_target_destroy(&bp->bif_psref, bpf_psref_class);
2270
2271 while (!SLIST_EMPTY(&bp->bif_trackers)) {
2272 struct bpf_event_tracker *t =
2273 SLIST_FIRST(&bp->bif_trackers);
2274 SLIST_REMOVE_HEAD(&bp->bif_trackers,
2275 bet_entries);
2276 kmem_free(t, sizeof(*t));
2277 }
2278
2279 BPF_IFLIST_ENTRY_DESTROY(bp);
2280 if (bp->bif_si != NULL) {
2281 /* XXX NOMPSAFE: assumed running on one CPU */
2282 s = splnet();
2283 while (bp->bif_mbuf_head != NULL) {
2284 struct mbuf *m = bp->bif_mbuf_head;
2285 bp->bif_mbuf_head = m->m_nextpkt;
2286 m_freem(m);
2287 }
2288 splx(s);
2289 softint_disestablish(bp->bif_si);
2290 }
2291 kmem_free(bp, sizeof(*bp));
2292 goto again;
2293 }
2294 }
2295 mutex_exit(&bpf_mtx);
2296 }
2297
2298 /*
2299 * Change the data link type of a interface.
2300 */
2301 static void
_bpf_change_type(struct ifnet * ifp,u_int dlt,u_int hdrlen)2302 _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen)
2303 {
2304 struct bpf_if *bp;
2305
2306 mutex_enter(&bpf_mtx);
2307 BPF_IFLIST_WRITER_FOREACH(bp) {
2308 if (bp->bif_driverp == &ifp->if_bpf)
2309 break;
2310 }
2311 if (bp == NULL)
2312 panic("bpf_change_type");
2313
2314 bp->bif_dlt = dlt;
2315
2316 bp->bif_hdrlen = hdrlen;
2317 mutex_exit(&bpf_mtx);
2318 }
2319
2320 /*
2321 * Get a list of available data link type of the interface.
2322 */
2323 static int
bpf_getdltlist(struct bpf_d * d,struct bpf_dltlist * bfl)2324 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl)
2325 {
2326 int n, error;
2327 struct ifnet *ifp;
2328 struct bpf_if *bp;
2329 int s, bound;
2330
2331 KASSERT(mutex_owned(d->bd_mtx));
2332
2333 ifp = d->bd_bif->bif_ifp;
2334 n = 0;
2335 error = 0;
2336
2337 bound = curlwp_bind();
2338 s = pserialize_read_enter();
2339 BPF_IFLIST_READER_FOREACH(bp) {
2340 if (bp->bif_ifp != ifp)
2341 continue;
2342 if (bfl->bfl_list != NULL) {
2343 struct psref psref;
2344
2345 if (n >= bfl->bfl_len) {
2346 pserialize_read_exit(s);
2347 return ENOMEM;
2348 }
2349
2350 bpf_if_acquire(bp, &psref);
2351 pserialize_read_exit(s);
2352
2353 error = copyout(&bp->bif_dlt,
2354 bfl->bfl_list + n, sizeof(u_int));
2355
2356 s = pserialize_read_enter();
2357 bpf_if_release(bp, &psref);
2358 }
2359 n++;
2360 }
2361 pserialize_read_exit(s);
2362 curlwp_bindx(bound);
2363
2364 bfl->bfl_len = n;
2365 return error;
2366 }
2367
2368 /*
2369 * Set the data link type of a BPF instance.
2370 */
2371 static int
bpf_setdlt(struct bpf_d * d,u_int dlt)2372 bpf_setdlt(struct bpf_d *d, u_int dlt)
2373 {
2374 int error, opromisc;
2375 struct ifnet *ifp;
2376 struct bpf_if *bp;
2377
2378 KASSERT(mutex_owned(&bpf_mtx));
2379 KASSERT(mutex_owned(d->bd_mtx));
2380
2381 if (d->bd_bif->bif_dlt == dlt)
2382 return 0;
2383 ifp = d->bd_bif->bif_ifp;
2384 BPF_IFLIST_WRITER_FOREACH(bp) {
2385 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt)
2386 break;
2387 }
2388 if (bp == NULL)
2389 return EINVAL;
2390 opromisc = d->bd_promisc;
2391 bpf_detachd(d);
2392 BPFIF_DLIST_ENTRY_INIT(d);
2393 bpf_attachd(d, bp);
2394 reset_d(d);
2395 if (opromisc) {
2396 KERNEL_LOCK_UNLESS_NET_MPSAFE();
2397 error = ifpromisc(bp->bif_ifp, 1);
2398 KERNEL_UNLOCK_UNLESS_NET_MPSAFE();
2399 if (error)
2400 printf("%s: bpf_setdlt: ifpromisc failed (%d)\n",
2401 bp->bif_ifp->if_xname, error);
2402 else
2403 d->bd_promisc = 1;
2404 }
2405 return 0;
2406 }
2407
2408 static int
sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS)2409 sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS)
2410 {
2411 int newsize, error;
2412 struct sysctlnode node;
2413
2414 node = *rnode;
2415 node.sysctl_data = &newsize;
2416 newsize = bpf_maxbufsize;
2417 error = sysctl_lookup(SYSCTLFN_CALL(&node));
2418 if (error || newp == NULL)
2419 return (error);
2420
2421 if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE)
2422 return (EINVAL);
2423
2424 bpf_maxbufsize = newsize;
2425
2426 return (0);
2427 }
2428
2429 #if defined(MODULAR) || defined(BPFJIT)
2430 static int
sysctl_net_bpf_jit(SYSCTLFN_ARGS)2431 sysctl_net_bpf_jit(SYSCTLFN_ARGS)
2432 {
2433 bool newval;
2434 int error;
2435 struct sysctlnode node;
2436
2437 node = *rnode;
2438 node.sysctl_data = &newval;
2439 newval = bpf_jit;
2440 error = sysctl_lookup(SYSCTLFN_CALL(&node));
2441 if (error != 0 || newp == NULL)
2442 return error;
2443
2444 bpf_jit = newval;
2445 if (newval && bpfjit_module_ops.bj_generate_code == NULL) {
2446 printf("JIT compilation is postponed "
2447 "until after bpfjit module is loaded\n");
2448 }
2449
2450 return 0;
2451 }
2452 #endif
2453
2454 static int
sysctl_net_bpf_peers(SYSCTLFN_ARGS)2455 sysctl_net_bpf_peers(SYSCTLFN_ARGS)
2456 {
2457 int error, elem_count;
2458 struct bpf_d *dp;
2459 struct bpf_d_ext dpe;
2460 size_t len, needed, elem_size, out_size;
2461 char *sp;
2462
2463 if (namelen == 1 && name[0] == CTL_QUERY)
2464 return (sysctl_query(SYSCTLFN_CALL(rnode)));
2465
2466 if (namelen != 2)
2467 return (EINVAL);
2468
2469 /* BPF peers is privileged information. */
2470 error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE,
2471 KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL);
2472 if (error)
2473 return (EPERM);
2474
2475 len = (oldp != NULL) ? *oldlenp : 0;
2476 sp = oldp;
2477 elem_size = name[0];
2478 elem_count = name[1];
2479 out_size = MIN(sizeof(dpe), elem_size);
2480 needed = 0;
2481
2482 if (elem_size < 1 || elem_count < 0)
2483 return (EINVAL);
2484
2485 mutex_enter(&bpf_mtx);
2486 BPF_DLIST_WRITER_FOREACH(dp) {
2487 if (len >= elem_size && elem_count > 0) {
2488 #define BPF_EXT(field) dpe.bde_ ## field = dp->bd_ ## field
2489 BPF_EXT(bufsize);
2490 BPF_EXT(promisc);
2491 BPF_EXT(state);
2492 BPF_EXT(immediate);
2493 BPF_EXT(hdrcmplt);
2494 BPF_EXT(direction);
2495 BPF_EXT(pid);
2496 BPF_EXT(rcount);
2497 BPF_EXT(dcount);
2498 BPF_EXT(ccount);
2499 #undef BPF_EXT
2500 mutex_enter(dp->bd_mtx);
2501 if (dp->bd_bif)
2502 (void)strlcpy(dpe.bde_ifname,
2503 dp->bd_bif->bif_ifp->if_xname,
2504 IFNAMSIZ - 1);
2505 else
2506 dpe.bde_ifname[0] = '\0';
2507 dpe.bde_locked = dp->bd_locked;
2508 mutex_exit(dp->bd_mtx);
2509
2510 error = copyout(&dpe, sp, out_size);
2511 if (error)
2512 break;
2513 sp += elem_size;
2514 len -= elem_size;
2515 }
2516 needed += elem_size;
2517 if (elem_count > 0 && elem_count != INT_MAX)
2518 elem_count--;
2519 }
2520 mutex_exit(&bpf_mtx);
2521
2522 *oldlenp = needed;
2523
2524 return (error);
2525 }
2526
2527 static void
bpf_stats(void * p,void * arg,struct cpu_info * ci __unused)2528 bpf_stats(void *p, void *arg, struct cpu_info *ci __unused)
2529 {
2530 struct bpf_stat *const stats = p;
2531 struct bpf_stat *sum = arg;
2532
2533 int s = splnet();
2534
2535 sum->bs_recv += stats->bs_recv;
2536 sum->bs_drop += stats->bs_drop;
2537 sum->bs_capt += stats->bs_capt;
2538
2539 splx(s);
2540 }
2541
2542 static int
bpf_sysctl_gstats_handler(SYSCTLFN_ARGS)2543 bpf_sysctl_gstats_handler(SYSCTLFN_ARGS)
2544 {
2545 struct sysctlnode node;
2546 int error;
2547 struct bpf_stat sum;
2548
2549 memset(&sum, 0, sizeof(sum));
2550 node = *rnode;
2551
2552 percpu_foreach_xcall(bpf_gstats_percpu, XC_HIGHPRI_IPL(IPL_SOFTNET),
2553 bpf_stats, &sum);
2554
2555 node.sysctl_data = ∑
2556 node.sysctl_size = sizeof(sum);
2557 error = sysctl_lookup(SYSCTLFN_CALL(&node));
2558 if (error != 0 || newp == NULL)
2559 return error;
2560
2561 return 0;
2562 }
2563
2564 SYSCTL_SETUP(sysctl_net_bpf_setup, "bpf sysctls")
2565 {
2566 const struct sysctlnode *node;
2567
2568 node = NULL;
2569 sysctl_createv(clog, 0, NULL, &node,
2570 CTLFLAG_PERMANENT,
2571 CTLTYPE_NODE, "bpf",
2572 SYSCTL_DESCR("BPF options"),
2573 NULL, 0, NULL, 0,
2574 CTL_NET, CTL_CREATE, CTL_EOL);
2575 if (node != NULL) {
2576 #if defined(MODULAR) || defined(BPFJIT)
2577 sysctl_createv(clog, 0, NULL, NULL,
2578 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
2579 CTLTYPE_BOOL, "jit",
2580 SYSCTL_DESCR("Toggle Just-In-Time compilation"),
2581 sysctl_net_bpf_jit, 0, &bpf_jit, 0,
2582 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
2583 #endif
2584 sysctl_createv(clog, 0, NULL, NULL,
2585 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
2586 CTLTYPE_INT, "maxbufsize",
2587 SYSCTL_DESCR("Maximum size for data capture buffer"),
2588 sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0,
2589 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
2590 sysctl_createv(clog, 0, NULL, NULL,
2591 CTLFLAG_PERMANENT,
2592 CTLTYPE_STRUCT, "stats",
2593 SYSCTL_DESCR("BPF stats"),
2594 bpf_sysctl_gstats_handler, 0, NULL, 0,
2595 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
2596 sysctl_createv(clog, 0, NULL, NULL,
2597 CTLFLAG_PERMANENT,
2598 CTLTYPE_STRUCT, "peers",
2599 SYSCTL_DESCR("BPF peers"),
2600 sysctl_net_bpf_peers, 0, NULL, 0,
2601 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
2602 }
2603
2604 }
2605
2606 static int
_bpf_register_track_event(struct bpf_if ** driverp,void (* _fun)(struct bpf_if *,struct ifnet *,int,int))2607 _bpf_register_track_event(struct bpf_if **driverp,
2608 void (*_fun)(struct bpf_if *, struct ifnet *, int, int))
2609 {
2610 struct bpf_if *bp;
2611 struct bpf_event_tracker *t;
2612 int ret = ENOENT;
2613
2614 t = kmem_zalloc(sizeof(*t), KM_SLEEP);
2615 if (!t)
2616 return ENOMEM;
2617 t->bet_notify = _fun;
2618
2619 mutex_enter(&bpf_mtx);
2620 BPF_IFLIST_WRITER_FOREACH(bp) {
2621 if (bp->bif_driverp != driverp)
2622 continue;
2623 SLIST_INSERT_HEAD(&bp->bif_trackers, t, bet_entries);
2624 ret = 0;
2625 break;
2626 }
2627 mutex_exit(&bpf_mtx);
2628
2629 return ret;
2630 }
2631
2632 static int
_bpf_deregister_track_event(struct bpf_if ** driverp,void (* _fun)(struct bpf_if *,struct ifnet *,int,int))2633 _bpf_deregister_track_event(struct bpf_if **driverp,
2634 void (*_fun)(struct bpf_if *, struct ifnet *, int, int))
2635 {
2636 struct bpf_if *bp;
2637 struct bpf_event_tracker *t = NULL;
2638 int ret = ENOENT;
2639
2640 mutex_enter(&bpf_mtx);
2641 BPF_IFLIST_WRITER_FOREACH(bp) {
2642 if (bp->bif_driverp != driverp)
2643 continue;
2644 SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) {
2645 if (t->bet_notify == _fun) {
2646 ret = 0;
2647 break;
2648 }
2649 }
2650 if (ret == 0)
2651 break;
2652 }
2653 if (ret == 0 && t && t->bet_notify == _fun) {
2654 SLIST_REMOVE(&bp->bif_trackers, t, bpf_event_tracker,
2655 bet_entries);
2656 }
2657 mutex_exit(&bpf_mtx);
2658 if (ret == 0)
2659 kmem_free(t, sizeof(*t));
2660 return ret;
2661 }
2662
2663 struct bpf_ops bpf_ops_kernel = {
2664 .bpf_attach = _bpfattach,
2665 .bpf_detach = _bpfdetach,
2666 .bpf_change_type = _bpf_change_type,
2667 .bpf_register_track_event = _bpf_register_track_event,
2668 .bpf_deregister_track_event = _bpf_deregister_track_event,
2669
2670 .bpf_mtap = _bpf_mtap,
2671 .bpf_mtap2 = _bpf_mtap2,
2672 .bpf_mtap_af = _bpf_mtap_af,
2673 .bpf_mtap_sl_in = _bpf_mtap_sl_in,
2674 .bpf_mtap_sl_out = _bpf_mtap_sl_out,
2675
2676 .bpf_mtap_softint = _bpf_mtap_softint,
2677 .bpf_mtap_softint_init = _bpf_mtap_softint_init,
2678 };
2679
2680 MODULE(MODULE_CLASS_DRIVER, bpf, "bpf_filter");
2681
2682 static int
bpf_modcmd(modcmd_t cmd,void * arg)2683 bpf_modcmd(modcmd_t cmd, void *arg)
2684 {
2685 #ifdef _MODULE
2686 devmajor_t bmajor, cmajor;
2687 #endif
2688 int error = 0;
2689
2690 switch (cmd) {
2691 case MODULE_CMD_INIT:
2692 bpf_init();
2693 #ifdef _MODULE
2694 bmajor = cmajor = NODEVMAJOR;
2695 error = devsw_attach("bpf", NULL, &bmajor,
2696 &bpf_cdevsw, &cmajor);
2697 if (error)
2698 break;
2699 #endif
2700
2701 bpf_ops_handover_enter(&bpf_ops_kernel);
2702 atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel);
2703 bpf_ops_handover_exit();
2704 break;
2705
2706 case MODULE_CMD_FINI:
2707 /*
2708 * While there is no reference counting for bpf callers,
2709 * unload could at least in theory be done similarly to
2710 * system call disestablishment. This should even be
2711 * a little simpler:
2712 *
2713 * 1) replace op vector with stubs
2714 * 2) post update to all cpus with xc
2715 * 3) check that nobody is in bpf anymore
2716 * (it's doubtful we'd want something like l_sysent,
2717 * but we could do something like *signed* percpu
2718 * counters. if the sum is 0, we're good).
2719 * 4) if fail, unroll changes
2720 *
2721 * NOTE: change won't be atomic to the outside. some
2722 * packets may be not captured even if unload is
2723 * not successful. I think packet capture not working
2724 * is a perfectly logical consequence of trying to
2725 * disable packet capture.
2726 */
2727 error = EOPNOTSUPP;
2728 break;
2729
2730 default:
2731 error = ENOTTY;
2732 break;
2733 }
2734
2735 return error;
2736 }
2737