Changeset View
Changeset View
Standalone View
Standalone View
head/sys/net/bpf.c
/*- | /*- | ||||
* SPDX-License-Identifier: BSD-3-Clause | * SPDX-License-Identifier: BSD-3-Clause | ||||
* | * | ||||
* Copyright (c) 1990, 1991, 1993 | * Copyright (c) 1990, 1991, 1993 | ||||
* The Regents of the University of California. All rights reserved. | * The Regents of the University of California. All rights reserved. | ||||
* Copyright (c) 2019 Andrey V. Elsukov <ae@FreeBSD.org> | |||||
* | * | ||||
* This code is derived from the Stanford/CMU enet packet filter, | * This code is derived from the Stanford/CMU enet packet filter, | ||||
* (net/enet.c) distributed as part of 4.3BSD, and code contributed | * (net/enet.c) distributed as part of 4.3BSD, and code contributed | ||||
* to Berkeley by Steven McCanne and Van Jacobson both of Lawrence | * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence | ||||
* Berkeley Laboratory. | * Berkeley Laboratory. | ||||
* | * | ||||
* Redistribution and use in source and binary forms, with or without | * Redistribution and use in source and binary forms, with or without | ||||
* modification, are permitted provided that the following conditions | * modification, are permitted provided that the following conditions | ||||
Show All 27 Lines | |||||
#include "opt_bpf.h" | #include "opt_bpf.h" | ||||
#include "opt_ddb.h" | #include "opt_ddb.h" | ||||
#include "opt_netgraph.h" | #include "opt_netgraph.h" | ||||
#include <sys/types.h> | #include <sys/types.h> | ||||
#include <sys/param.h> | #include <sys/param.h> | ||||
#include <sys/lock.h> | #include <sys/lock.h> | ||||
#include <sys/rwlock.h> | |||||
#include <sys/systm.h> | #include <sys/systm.h> | ||||
#include <sys/conf.h> | #include <sys/conf.h> | ||||
#include <sys/fcntl.h> | #include <sys/fcntl.h> | ||||
#include <sys/jail.h> | #include <sys/jail.h> | ||||
#include <sys/malloc.h> | #include <sys/malloc.h> | ||||
#include <sys/mbuf.h> | #include <sys/mbuf.h> | ||||
#include <sys/time.h> | #include <sys/time.h> | ||||
#include <sys/priv.h> | #include <sys/priv.h> | ||||
Show All 36 Lines | |||||
#include <net80211/ieee80211_freebsd.h> | #include <net80211/ieee80211_freebsd.h> | ||||
#include <security/mac/mac_framework.h> | #include <security/mac/mac_framework.h> | ||||
MALLOC_DEFINE(M_BPF, "BPF", "BPF data"); | MALLOC_DEFINE(M_BPF, "BPF", "BPF data"); | ||||
static struct bpf_if_ext dead_bpf_if = { | static struct bpf_if_ext dead_bpf_if = { | ||||
.bif_dlist = LIST_HEAD_INITIALIZER() | .bif_dlist = CK_LIST_HEAD_INITIALIZER() | ||||
}; | }; | ||||
struct bpf_if { | struct bpf_if { | ||||
#define bif_next bif_ext.bif_next | #define bif_next bif_ext.bif_next | ||||
#define bif_dlist bif_ext.bif_dlist | #define bif_dlist bif_ext.bif_dlist | ||||
struct bpf_if_ext bif_ext; /* public members */ | struct bpf_if_ext bif_ext; /* public members */ | ||||
u_int bif_dlt; /* link layer type */ | u_int bif_dlt; /* link layer type */ | ||||
u_int bif_hdrlen; /* length of link header */ | u_int bif_hdrlen; /* length of link header */ | ||||
struct bpfd_list bif_wlist; /* writer-only list */ | |||||
struct ifnet *bif_ifp; /* corresponding interface */ | struct ifnet *bif_ifp; /* corresponding interface */ | ||||
struct rwlock bif_lock; /* interface lock */ | |||||
LIST_HEAD(, bpf_d) bif_wlist; /* writer-only list */ | |||||
int bif_flags; /* Interface flags */ | |||||
struct bpf_if **bif_bpf; /* Pointer to pointer to us */ | struct bpf_if **bif_bpf; /* Pointer to pointer to us */ | ||||
volatile u_int bif_refcnt; | |||||
struct epoch_context epoch_ctx; | |||||
}; | }; | ||||
CTASSERT(offsetof(struct bpf_if, bif_ext) == 0); | CTASSERT(offsetof(struct bpf_if, bif_ext) == 0); | ||||
#define BPFIF_RLOCK(bif) rw_rlock(&(bif)->bif_lock) | struct bpf_program_buffer { | ||||
#define BPFIF_RUNLOCK(bif) rw_runlock(&(bif)->bif_lock) | struct epoch_context epoch_ctx; | ||||
#define BPFIF_WLOCK(bif) rw_wlock(&(bif)->bif_lock) | #ifdef BPF_JITTER | ||||
#define BPFIF_WUNLOCK(bif) rw_wunlock(&(bif)->bif_lock) | bpf_jit_filter *func; | ||||
#endif | |||||
void *buffer[0]; | |||||
}; | |||||
#if defined(DEV_BPF) || defined(NETGRAPH_BPF) | #if defined(DEV_BPF) || defined(NETGRAPH_BPF) | ||||
#define PRINET 26 /* interruptible */ | #define PRINET 26 /* interruptible */ | ||||
#define SIZEOF_BPF_HDR(type) \ | #define SIZEOF_BPF_HDR(type) \ | ||||
(offsetof(type, bh_hdrlen) + sizeof(((type *)0)->bh_hdrlen)) | (offsetof(type, bh_hdrlen) + sizeof(((type *)0)->bh_hdrlen)) | ||||
Show All 36 Lines | |||||
#define BIOCSETFNR32 _IOW('B', 130, struct bpf_program32) | #define BIOCSETFNR32 _IOW('B', 130, struct bpf_program32) | ||||
#endif | #endif | ||||
#define BPF_LOCK() sx_xlock(&bpf_sx) | #define BPF_LOCK() sx_xlock(&bpf_sx) | ||||
#define BPF_UNLOCK() sx_xunlock(&bpf_sx) | #define BPF_UNLOCK() sx_xunlock(&bpf_sx) | ||||
#define BPF_LOCK_ASSERT() sx_assert(&bpf_sx, SA_XLOCKED) | #define BPF_LOCK_ASSERT() sx_assert(&bpf_sx, SA_XLOCKED) | ||||
/* | /* | ||||
* bpf_iflist is a list of BPF interface structures, each corresponding to a | * bpf_iflist is a list of BPF interface structures, each corresponding to a | ||||
* specific DLT. The same network interface might have several BPF interface | * specific DLT. The same network interface might have several BPF interface | ||||
* structures registered by different layers in the stack (i.e., 802.11 | * structures registered by different layers in the stack (i.e., 802.11 | ||||
* frames, ethernet frames, etc). | * frames, ethernet frames, etc). | ||||
*/ | */ | ||||
static LIST_HEAD(, bpf_if) bpf_iflist, bpf_freelist; | CK_LIST_HEAD(bpf_iflist, bpf_if); | ||||
static struct bpf_iflist bpf_iflist; | |||||
static struct sx bpf_sx; /* bpf global lock */ | static struct sx bpf_sx; /* bpf global lock */ | ||||
static int bpf_bpfd_cnt; | static int bpf_bpfd_cnt; | ||||
static void bpfif_ref(struct bpf_if *); | |||||
static void bpfif_rele(struct bpf_if *); | |||||
static void bpfd_ref(struct bpf_d *); | |||||
static void bpfd_rele(struct bpf_d *); | |||||
static void bpf_attachd(struct bpf_d *, struct bpf_if *); | static void bpf_attachd(struct bpf_d *, struct bpf_if *); | ||||
static void bpf_detachd(struct bpf_d *); | static void bpf_detachd(struct bpf_d *); | ||||
static void bpf_detachd_locked(struct bpf_d *); | static void bpf_detachd_locked(struct bpf_d *, bool); | ||||
static void bpf_freed(struct bpf_d *); | static void bpfd_free(epoch_context_t); | ||||
static int bpf_movein(struct uio *, int, struct ifnet *, struct mbuf **, | static int bpf_movein(struct uio *, int, struct ifnet *, struct mbuf **, | ||||
struct sockaddr *, int *, struct bpf_d *); | struct sockaddr *, int *, struct bpf_d *); | ||||
static int bpf_setif(struct bpf_d *, struct ifreq *); | static int bpf_setif(struct bpf_d *, struct ifreq *); | ||||
static void bpf_timed_out(void *); | static void bpf_timed_out(void *); | ||||
static __inline void | static __inline void | ||||
bpf_wakeup(struct bpf_d *); | bpf_wakeup(struct bpf_d *); | ||||
static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, | static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, | ||||
void (*)(struct bpf_d *, caddr_t, u_int, void *, u_int), | void (*)(struct bpf_d *, caddr_t, u_int, void *, u_int), | ||||
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines | |||||
}; | }; | ||||
static struct filterops bpfread_filtops = { | static struct filterops bpfread_filtops = { | ||||
.f_isfd = 1, | .f_isfd = 1, | ||||
.f_detach = filt_bpfdetach, | .f_detach = filt_bpfdetach, | ||||
.f_event = filt_bpfread, | .f_event = filt_bpfread, | ||||
}; | }; | ||||
eventhandler_tag bpf_ifdetach_cookie = NULL; | |||||
/* | /* | ||||
* LOCKING MODEL USED BY BPF: | * LOCKING MODEL USED BY BPF | ||||
* | |||||
* Locks: | * Locks: | ||||
* 1) global lock (BPF_LOCK). Mutex, used to protect interface addition/removal, | * 1) global lock (BPF_LOCK). Sx, used to protect some global counters, | ||||
* some global counters and every bpf_if reference. | * every bpf_iflist changes, serializes ioctl access to bpf descriptors. | ||||
* 2) Interface lock. Rwlock, used to protect list of BPF descriptors and their filters. | * 2) Descriptor lock. Mutex, used to protect BPF buffers and various | ||||
* 3) Descriptor lock. Mutex, used to protect BPF buffers and various structure fields | * structure fields used by bpf_*tap* code. | ||||
* used by bpf_mtap code. | |||||
* | * | ||||
* Lock order: | * Lock order: global lock, then descriptor lock. | ||||
* | * | ||||
* Global lock, interface lock, descriptor lock | * There are several possible consumers: | ||||
* | * | ||||
* We have to acquire interface lock before descriptor main lock due to BPF_MTAP[2] | * 1. The kernel registers interface pointer with bpfattach(). | ||||
* working model. In many places (like bpf_detachd) we start with BPF descriptor | * Each call allocates new bpf_if structure, references ifnet pointer | ||||
* (and we need to at least rlock it to get reliable interface pointer). This | * and links bpf_if into bpf_iflist chain. This is protected with global | ||||
* gives us potential LOR. As a result, we use global lock to protect from bpf_if | * lock. | ||||
* change in every such place. | |||||
* | * | ||||
* Changing d->bd_bif is protected by 1) global lock, 2) interface lock and | * 2. An userland application uses ioctl() call to bpf_d descriptor. | ||||
* 3) descriptor main wlock. | * All such call are serialized with global lock. BPF filters can be | ||||
* Reading bd_bif can be protected by any of these locks, typically global lock. | * changed, but pointer to old filter will be freed using epoch_call(). | ||||
* Thus it should be safe for bpf_tap/bpf_mtap* code to do access to | |||||
* filter pointers, even if change will happen during bpf_tap execution. | |||||
* Destroying of bpf_d descriptor also is doing using epoch_call(). | |||||
* | * | ||||
* Changing read/write BPF filter is protected by the same three locks, | * 3. An userland application can write packets into bpf_d descriptor. | ||||
* the same applies for reading. | * There we need to be sure, that ifnet won't disappear during bpfwrite(). | ||||
* | * | ||||
* Sleeping in global lock is not allowed due to bpfdetach() using it. | * 4. The kernel invokes bpf_tap/bpf_mtap* functions. The access to | ||||
* bif_dlist is protected with net_epoch_preempt section. So, it should | |||||
* be safe to make access to bpf_d descriptor inside the section. | |||||
* | |||||
* 5. The kernel invokes bpfdetach() on interface destroying. All lists | |||||
* are modified with global lock held and actual free() is done using | |||||
* epoch_call(). | |||||
*/ | */ | ||||
static void | |||||
bpfif_free(epoch_context_t ctx) | |||||
{ | |||||
struct bpf_if *bp; | |||||
bp = __containerof(ctx, struct bpf_if, epoch_ctx); | |||||
if_rele(bp->bif_ifp); | |||||
free(bp, M_BPF); | |||||
} | |||||
static void | |||||
bpfif_ref(struct bpf_if *bp) | |||||
{ | |||||
refcount_acquire(&bp->bif_refcnt); | |||||
} | |||||
static void | |||||
bpfif_rele(struct bpf_if *bp) | |||||
{ | |||||
if (!refcount_release(&bp->bif_refcnt)) | |||||
return; | |||||
epoch_call(net_epoch_preempt, &bp->epoch_ctx, bpfif_free); | |||||
} | |||||
static void | |||||
bpfd_ref(struct bpf_d *d) | |||||
{ | |||||
refcount_acquire(&d->bd_refcnt); | |||||
} | |||||
static void | |||||
bpfd_rele(struct bpf_d *d) | |||||
{ | |||||
if (!refcount_release(&d->bd_refcnt)) | |||||
return; | |||||
epoch_call(net_epoch_preempt, &d->epoch_ctx, bpfd_free); | |||||
} | |||||
static struct bpf_program_buffer* | |||||
bpf_program_buffer_alloc(size_t size, int flags) | |||||
{ | |||||
return (malloc(sizeof(struct bpf_program_buffer) + size, | |||||
M_BPF, flags)); | |||||
} | |||||
static void | |||||
bpf_program_buffer_free(epoch_context_t ctx) | |||||
{ | |||||
struct bpf_program_buffer *ptr; | |||||
ptr = __containerof(ctx, struct bpf_program_buffer, epoch_ctx); | |||||
#ifdef BPF_JITTER | |||||
if (ptr->func != NULL) | |||||
bpf_destroy_jit_filter(ptr->func); | |||||
#endif | |||||
free(ptr, M_BPF); | |||||
} | |||||
/* | /* | ||||
* Wrapper functions for various buffering methods. If the set of buffer | * Wrapper functions for various buffering methods. If the set of buffer | ||||
* modes expands, we will probably want to introduce a switch data structure | * modes expands, we will probably want to introduce a switch data structure | ||||
* similar to protosw, et. | * similar to protosw, et. | ||||
*/ | */ | ||||
static void | static void | ||||
bpf_append_bytes(struct bpf_d *d, caddr_t buf, u_int offset, void *src, | bpf_append_bytes(struct bpf_d *d, caddr_t buf, u_int offset, void *src, | ||||
u_int len) | u_int len) | ||||
▲ Show 20 Lines • Show All 337 Lines • ▼ Show 20 Lines | bpf_movein(struct uio *uio, int linktype, struct ifnet *ifp, struct mbuf **mp, | ||||
return (0); | return (0); | ||||
bad: | bad: | ||||
m_freem(m); | m_freem(m); | ||||
return (error); | return (error); | ||||
} | } | ||||
/* | /* | ||||
* Attach file to the bpf interface, i.e. make d listen on bp. | * Attach descriptor to the bpf interface, i.e. make d listen on bp, | ||||
* then reset its buffers and counters with reset_d(). | |||||
*/ | */ | ||||
static void | static void | ||||
bpf_attachd(struct bpf_d *d, struct bpf_if *bp) | bpf_attachd(struct bpf_d *d, struct bpf_if *bp) | ||||
{ | { | ||||
int op_w; | int op_w; | ||||
BPF_LOCK_ASSERT(); | BPF_LOCK_ASSERT(); | ||||
/* | /* | ||||
* Save sysctl value to protect from sysctl change | * Save sysctl value to protect from sysctl change | ||||
* between reads | * between reads | ||||
*/ | */ | ||||
op_w = V_bpf_optimize_writers || d->bd_writer; | op_w = V_bpf_optimize_writers || d->bd_writer; | ||||
if (d->bd_bif != NULL) | if (d->bd_bif != NULL) | ||||
bpf_detachd_locked(d); | bpf_detachd_locked(d, false); | ||||
/* | /* | ||||
* Point d at bp, and add d to the interface's list. | * Point d at bp, and add d to the interface's list. | ||||
* Since there are many applications using BPF for | * Since there are many applications using BPF for | ||||
* sending raw packets only (dhcpd, cdpd are good examples) | * sending raw packets only (dhcpd, cdpd are good examples) | ||||
* we can delay adding d to the list of active listeners until | * we can delay adding d to the list of active listeners until | ||||
* some filter is configured. | * some filter is configured. | ||||
*/ | */ | ||||
BPFIF_WLOCK(bp); | |||||
BPFD_LOCK(d); | BPFD_LOCK(d); | ||||
/* | |||||
* Hold reference to bpif while descriptor uses this interface. | |||||
*/ | |||||
bpfif_ref(bp); | |||||
d->bd_bif = bp; | d->bd_bif = bp; | ||||
if (op_w != 0) { | if (op_w != 0) { | ||||
/* Add to writers-only list */ | /* Add to writers-only list */ | ||||
LIST_INSERT_HEAD(&bp->bif_wlist, d, bd_next); | CK_LIST_INSERT_HEAD(&bp->bif_wlist, d, bd_next); | ||||
/* | /* | ||||
* We decrement bd_writer on every filter set operation. | * We decrement bd_writer on every filter set operation. | ||||
* First BIOCSETF is done by pcap_open_live() to set up | * First BIOCSETF is done by pcap_open_live() to set up | ||||
* snap length. After that appliation usually sets its own filter | * snap length. After that appliation usually sets its own | ||||
* filter. | |||||
*/ | */ | ||||
d->bd_writer = 2; | d->bd_writer = 2; | ||||
} else | } else | ||||
LIST_INSERT_HEAD(&bp->bif_dlist, d, bd_next); | CK_LIST_INSERT_HEAD(&bp->bif_dlist, d, bd_next); | ||||
reset_d(d); | |||||
BPFD_UNLOCK(d); | BPFD_UNLOCK(d); | ||||
BPFIF_WUNLOCK(bp); | |||||
bpf_bpfd_cnt++; | bpf_bpfd_cnt++; | ||||
CTR3(KTR_NET, "%s: bpf_attach called by pid %d, adding to %s list", | CTR3(KTR_NET, "%s: bpf_attach called by pid %d, adding to %s list", | ||||
__func__, d->bd_pid, d->bd_writer ? "writer" : "active"); | __func__, d->bd_pid, d->bd_writer ? "writer" : "active"); | ||||
if (op_w == 0) | if (op_w == 0) | ||||
EVENTHANDLER_INVOKE(bpf_track, bp->bif_ifp, bp->bif_dlt, 1); | EVENTHANDLER_INVOKE(bpf_track, bp->bif_ifp, bp->bif_dlt, 1); | ||||
} | } | ||||
/* | /* | ||||
* Check if we need to upgrade our descriptor @d from write-only mode. | * Check if we need to upgrade our descriptor @d from write-only mode. | ||||
*/ | */ | ||||
static int | static int | ||||
bpf_check_upgrade(u_long cmd, struct bpf_d *d, struct bpf_insn *fcode, int flen) | bpf_check_upgrade(u_long cmd, struct bpf_d *d, struct bpf_insn *fcode, | ||||
int flen) | |||||
{ | { | ||||
int is_snap, need_upgrade; | int is_snap, need_upgrade; | ||||
/* | /* | ||||
* Check if we've already upgraded or new filter is empty. | * Check if we've already upgraded or new filter is empty. | ||||
*/ | */ | ||||
if (d->bd_writer == 0 || fcode == NULL) | if (d->bd_writer == 0 || fcode == NULL) | ||||
return (0); | return (0); | ||||
need_upgrade = 0; | need_upgrade = 0; | ||||
/* | /* | ||||
* Check if cmd looks like snaplen setting from | * Check if cmd looks like snaplen setting from | ||||
* pcap_bpf.c:pcap_open_live(). | * pcap_bpf.c:pcap_open_live(). | ||||
* Note we're not checking .k value here: | * Note we're not checking .k value here: | ||||
* while pcap_open_live() definitely sets to non-zero value, | * while pcap_open_live() definitely sets to non-zero value, | ||||
* we'd prefer to treat k=0 (deny ALL) case the same way: e.g. | * we'd prefer to treat k=0 (deny ALL) case the same way: e.g. | ||||
* do not consider upgrading immediately | * do not consider upgrading immediately | ||||
*/ | */ | ||||
if (cmd == BIOCSETF && flen == 1 && fcode[0].code == (BPF_RET | BPF_K)) | if (cmd == BIOCSETF && flen == 1 && | ||||
fcode[0].code == (BPF_RET | BPF_K)) | |||||
is_snap = 1; | is_snap = 1; | ||||
else | else | ||||
is_snap = 0; | is_snap = 0; | ||||
if (is_snap == 0) { | if (is_snap == 0) { | ||||
/* | /* | ||||
* We're setting first filter and it doesn't look like | * We're setting first filter and it doesn't look like | ||||
* setting snaplen. We're probably using bpf directly. | * setting snaplen. We're probably using bpf directly. | ||||
Show All 21 Lines | CTR5(KTR_NET, | ||||
"bd_writer counter %d, snap %d upgrade %d", | "bd_writer counter %d, snap %d upgrade %d", | ||||
__func__, d->bd_pid, d->bd_writer, | __func__, d->bd_pid, d->bd_writer, | ||||
is_snap, need_upgrade); | is_snap, need_upgrade); | ||||
return (need_upgrade); | return (need_upgrade); | ||||
} | } | ||||
/* | /* | ||||
* Add d to the list of active bp filters. | |||||
* Requires bpf_attachd() to be called before. | |||||
*/ | |||||
static void | |||||
bpf_upgraded(struct bpf_d *d) | |||||
{ | |||||
struct bpf_if *bp; | |||||
BPF_LOCK_ASSERT(); | |||||
bp = d->bd_bif; | |||||
/* | |||||
* Filter can be set several times without specifying interface. | |||||
* Mark d as reader and exit. | |||||
*/ | |||||
if (bp == NULL) { | |||||
BPFD_LOCK(d); | |||||
d->bd_writer = 0; | |||||
BPFD_UNLOCK(d); | |||||
return; | |||||
} | |||||
BPFIF_WLOCK(bp); | |||||
BPFD_LOCK(d); | |||||
/* Remove from writers-only list */ | |||||
LIST_REMOVE(d, bd_next); | |||||
LIST_INSERT_HEAD(&bp->bif_dlist, d, bd_next); | |||||
/* Mark d as reader */ | |||||
d->bd_writer = 0; | |||||
BPFD_UNLOCK(d); | |||||
BPFIF_WUNLOCK(bp); | |||||
CTR2(KTR_NET, "%s: upgrade required by pid %d", __func__, d->bd_pid); | |||||
EVENTHANDLER_INVOKE(bpf_track, bp->bif_ifp, bp->bif_dlt, 1); | |||||
} | |||||
/* | |||||
* Detach a file from its interface. | * Detach a file from its interface. | ||||
*/ | */ | ||||
static void | static void | ||||
bpf_detachd(struct bpf_d *d) | bpf_detachd(struct bpf_d *d) | ||||
{ | { | ||||
BPF_LOCK(); | BPF_LOCK(); | ||||
bpf_detachd_locked(d); | bpf_detachd_locked(d, false); | ||||
BPF_UNLOCK(); | BPF_UNLOCK(); | ||||
} | } | ||||
static void | static void | ||||
bpf_detachd_locked(struct bpf_d *d) | bpf_detachd_locked(struct bpf_d *d, bool detached_ifp) | ||||
{ | { | ||||
int error; | |||||
struct bpf_if *bp; | struct bpf_if *bp; | ||||
struct ifnet *ifp; | struct ifnet *ifp; | ||||
int error; | |||||
BPF_LOCK_ASSERT(); | |||||
CTR2(KTR_NET, "%s: detach required by pid %d", __func__, d->bd_pid); | CTR2(KTR_NET, "%s: detach required by pid %d", __func__, d->bd_pid); | ||||
BPF_LOCK_ASSERT(); | |||||
/* Check if descriptor is attached */ | /* Check if descriptor is attached */ | ||||
if ((bp = d->bd_bif) == NULL) | if ((bp = d->bd_bif) == NULL) | ||||
return; | return; | ||||
BPFIF_WLOCK(bp); | |||||
BPFD_LOCK(d); | BPFD_LOCK(d); | ||||
/* Remove d from the interface's descriptor list. */ | |||||
CK_LIST_REMOVE(d, bd_next); | |||||
/* Save bd_writer value */ | /* Save bd_writer value */ | ||||
error = d->bd_writer; | error = d->bd_writer; | ||||
/* | |||||
* Remove d from the interface's descriptor list. | |||||
*/ | |||||
LIST_REMOVE(d, bd_next); | |||||
ifp = bp->bif_ifp; | ifp = bp->bif_ifp; | ||||
d->bd_bif = NULL; | d->bd_bif = NULL; | ||||
if (detached_ifp) { | |||||
/* | |||||
* Notify descriptor as it's detached, so that any | |||||
* sleepers wake up and get ENXIO. | |||||
*/ | |||||
bpf_wakeup(d); | |||||
} | |||||
BPFD_UNLOCK(d); | BPFD_UNLOCK(d); | ||||
BPFIF_WUNLOCK(bp); | |||||
bpf_bpfd_cnt--; | bpf_bpfd_cnt--; | ||||
/* Call event handler iff d is attached */ | /* Call event handler iff d is attached */ | ||||
if (error == 0) | if (error == 0) | ||||
EVENTHANDLER_INVOKE(bpf_track, ifp, bp->bif_dlt, 0); | EVENTHANDLER_INVOKE(bpf_track, ifp, bp->bif_dlt, 0); | ||||
/* | /* | ||||
* Check if this descriptor had requested promiscuous mode. | * Check if this descriptor had requested promiscuous mode. | ||||
* If so, turn it off. | * If so and ifnet is not detached, turn it off. | ||||
*/ | */ | ||||
if (d->bd_promisc) { | if (d->bd_promisc && !detached_ifp) { | ||||
d->bd_promisc = 0; | d->bd_promisc = 0; | ||||
CURVNET_SET(ifp->if_vnet); | CURVNET_SET(ifp->if_vnet); | ||||
error = ifpromisc(ifp, 0); | error = ifpromisc(ifp, 0); | ||||
CURVNET_RESTORE(); | CURVNET_RESTORE(); | ||||
if (error != 0 && error != ENXIO) { | if (error != 0 && error != ENXIO) { | ||||
/* | /* | ||||
* ENXIO can happen if a pccard is unplugged | * ENXIO can happen if a pccard is unplugged | ||||
* Something is really wrong if we were able to put | * Something is really wrong if we were able to put | ||||
* the driver into promiscuous mode, but can't | * the driver into promiscuous mode, but can't | ||||
* take it out. | * take it out. | ||||
*/ | */ | ||||
if_printf(bp->bif_ifp, | if_printf(bp->bif_ifp, | ||||
"bpf_detach: ifpromisc failed (%d)\n", error); | "bpf_detach: ifpromisc failed (%d)\n", error); | ||||
} | } | ||||
} | } | ||||
bpfif_rele(bp); | |||||
} | } | ||||
/* | /* | ||||
* Close the descriptor by detaching it from its interface, | * Close the descriptor by detaching it from its interface, | ||||
* deallocating its buffers, and marking it free. | * deallocating its buffers, and marking it free. | ||||
*/ | */ | ||||
static void | static void | ||||
bpf_dtor(void *data) | bpf_dtor(void *data) | ||||
{ | { | ||||
struct bpf_d *d = data; | struct bpf_d *d = data; | ||||
BPFD_LOCK(d); | BPFD_LOCK(d); | ||||
if (d->bd_state == BPF_WAITING) | if (d->bd_state == BPF_WAITING) | ||||
callout_stop(&d->bd_callout); | callout_stop(&d->bd_callout); | ||||
d->bd_state = BPF_IDLE; | d->bd_state = BPF_IDLE; | ||||
BPFD_UNLOCK(d); | BPFD_UNLOCK(d); | ||||
funsetown(&d->bd_sigio); | funsetown(&d->bd_sigio); | ||||
bpf_detachd(d); | bpf_detachd(d); | ||||
#ifdef MAC | #ifdef MAC | ||||
mac_bpfdesc_destroy(d); | mac_bpfdesc_destroy(d); | ||||
#endif /* MAC */ | #endif /* MAC */ | ||||
seldrain(&d->bd_sel); | seldrain(&d->bd_sel); | ||||
knlist_destroy(&d->bd_sel.si_note); | knlist_destroy(&d->bd_sel.si_note); | ||||
callout_drain(&d->bd_callout); | callout_drain(&d->bd_callout); | ||||
bpf_freed(d); | bpfd_rele(d); | ||||
free(d, M_BPF); | |||||
} | } | ||||
/* | /* | ||||
* Open ethernet device. Returns ENXIO for illegal minor device number, | * Open ethernet device. Returns ENXIO for illegal minor device number, | ||||
* EBUSY if file is open by another process. | * EBUSY if file is open by another process. | ||||
*/ | */ | ||||
/* ARGSUSED */ | /* ARGSUSED */ | ||||
static int | static int | ||||
Show All 25 Lines | bpfopen(struct cdev *dev, int flags, int fmt, struct thread *td) | ||||
*/ | */ | ||||
bpf_buffer_init(d); | bpf_buffer_init(d); | ||||
if ((flags & FREAD) == 0) | if ((flags & FREAD) == 0) | ||||
d->bd_writer = 2; | d->bd_writer = 2; | ||||
d->bd_hbuf_in_use = 0; | d->bd_hbuf_in_use = 0; | ||||
d->bd_bufmode = BPF_BUFMODE_BUFFER; | d->bd_bufmode = BPF_BUFMODE_BUFFER; | ||||
d->bd_sig = SIGIO; | d->bd_sig = SIGIO; | ||||
d->bd_direction = BPF_D_INOUT; | d->bd_direction = BPF_D_INOUT; | ||||
d->bd_refcnt = 1; | |||||
BPF_PID_REFRESH(d, td); | BPF_PID_REFRESH(d, td); | ||||
#ifdef MAC | #ifdef MAC | ||||
mac_bpfdesc_init(d); | mac_bpfdesc_init(d); | ||||
mac_bpfdesc_create(td->td_ucred, d); | mac_bpfdesc_create(td->td_ucred, d); | ||||
#endif | #endif | ||||
mtx_init(&d->bd_lock, devtoname(dev), "bpf cdev lock", MTX_DEF); | mtx_init(&d->bd_lock, devtoname(dev), "bpf cdev lock", MTX_DEF); | ||||
callout_init_mtx(&d->bd_callout, &d->bd_lock, 0); | callout_init_mtx(&d->bd_callout, &d->bd_lock, 0); | ||||
knlist_init_mtx(&d->bd_sel.si_note, &d->bd_lock); | knlist_init_mtx(&d->bd_sel.si_note, &d->bd_lock); | ||||
▲ Show 20 Lines • Show All 159 Lines • ▼ Show 20 Lines | |||||
static void | static void | ||||
bpf_timed_out(void *arg) | bpf_timed_out(void *arg) | ||||
{ | { | ||||
struct bpf_d *d = (struct bpf_d *)arg; | struct bpf_d *d = (struct bpf_d *)arg; | ||||
BPFD_LOCK_ASSERT(d); | BPFD_LOCK_ASSERT(d); | ||||
if (callout_pending(&d->bd_callout) || !callout_active(&d->bd_callout)) | if (callout_pending(&d->bd_callout) || | ||||
!callout_active(&d->bd_callout)) | |||||
return; | return; | ||||
if (d->bd_state == BPF_WAITING) { | if (d->bd_state == BPF_WAITING) { | ||||
d->bd_state = BPF_TIMED_OUT; | d->bd_state = BPF_TIMED_OUT; | ||||
if (d->bd_slen != 0) | if (d->bd_slen != 0) | ||||
bpf_wakeup(d); | bpf_wakeup(d); | ||||
} | } | ||||
} | } | ||||
Show All 9 Lines | if ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && | ||||
d->bd_slen != 0) | d->bd_slen != 0) | ||||
return (1); | return (1); | ||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
bpfwrite(struct cdev *dev, struct uio *uio, int ioflag) | bpfwrite(struct cdev *dev, struct uio *uio, int ioflag) | ||||
{ | { | ||||
struct route ro; | |||||
struct sockaddr dst; | |||||
struct epoch_tracker et; | |||||
struct bpf_if *bp; | |||||
struct bpf_d *d; | struct bpf_d *d; | ||||
struct ifnet *ifp; | struct ifnet *ifp; | ||||
struct mbuf *m, *mc; | struct mbuf *m, *mc; | ||||
struct sockaddr dst; | |||||
struct route ro; | |||||
int error, hlen; | int error, hlen; | ||||
error = devfs_get_cdevpriv((void **)&d); | error = devfs_get_cdevpriv((void **)&d); | ||||
if (error != 0) | if (error != 0) | ||||
return (error); | return (error); | ||||
NET_EPOCH_ENTER(et); | |||||
BPFD_LOCK(d); | |||||
BPF_PID_REFRESH_CUR(d); | BPF_PID_REFRESH_CUR(d); | ||||
counter_u64_add(d->bd_wcount, 1); | counter_u64_add(d->bd_wcount, 1); | ||||
/* XXX: locking required */ | if ((bp = d->bd_bif) == NULL) { | ||||
if (d->bd_bif == NULL) { | error = ENXIO; | ||||
counter_u64_add(d->bd_wdcount, 1); | goto out_locked; | ||||
return (ENXIO); | |||||
} | } | ||||
ifp = d->bd_bif->bif_ifp; | ifp = bp->bif_ifp; | ||||
if ((ifp->if_flags & IFF_UP) == 0) { | if ((ifp->if_flags & IFF_UP) == 0) { | ||||
counter_u64_add(d->bd_wdcount, 1); | error = ENETDOWN; | ||||
return (ENETDOWN); | goto out_locked; | ||||
} | } | ||||
if (uio->uio_resid == 0) { | if (uio->uio_resid == 0) | ||||
counter_u64_add(d->bd_wdcount, 1); | goto out_locked; | ||||
return (0); | |||||
} | |||||
bzero(&dst, sizeof(dst)); | bzero(&dst, sizeof(dst)); | ||||
m = NULL; | m = NULL; | ||||
hlen = 0; | hlen = 0; | ||||
/* XXX: bpf_movein() can sleep */ | |||||
error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, ifp, | /* | ||||
* Take extra reference, unlock d and exit from epoch section, | |||||
* since bpf_movein() can sleep. | |||||
*/ | |||||
bpfd_ref(d); | |||||
NET_EPOCH_EXIT(et); | |||||
BPFD_UNLOCK(d); | |||||
error = bpf_movein(uio, (int)bp->bif_dlt, ifp, | |||||
&m, &dst, &hlen, d); | &m, &dst, &hlen, d); | ||||
if (error) { | |||||
if (error != 0) { | |||||
counter_u64_add(d->bd_wdcount, 1); | counter_u64_add(d->bd_wdcount, 1); | ||||
bpfd_rele(d); | |||||
return (error); | return (error); | ||||
} | } | ||||
BPFD_LOCK(d); | |||||
/* | |||||
* Check that descriptor is still attached to the interface. | |||||
* This can happen on bpfdetach(). To avoid access to detached | |||||
* ifnet, free mbuf and return ENXIO. | |||||
*/ | |||||
if (d->bd_bif == NULL) { | |||||
counter_u64_add(d->bd_wdcount, 1); | |||||
BPFD_UNLOCK(d); | |||||
bpfd_rele(d); | |||||
m_freem(m); | |||||
return (ENXIO); | |||||
} | |||||
counter_u64_add(d->bd_wfcount, 1); | counter_u64_add(d->bd_wfcount, 1); | ||||
if (d->bd_hdrcmplt) | if (d->bd_hdrcmplt) | ||||
dst.sa_family = pseudo_AF_HDRCMPLT; | dst.sa_family = pseudo_AF_HDRCMPLT; | ||||
if (d->bd_feedback) { | if (d->bd_feedback) { | ||||
mc = m_dup(m, M_NOWAIT); | mc = m_dup(m, M_NOWAIT); | ||||
if (mc != NULL) | if (mc != NULL) | ||||
mc->m_pkthdr.rcvif = ifp; | mc->m_pkthdr.rcvif = ifp; | ||||
/* Set M_PROMISC for outgoing packets to be discarded. */ | /* Set M_PROMISC for outgoing packets to be discarded. */ | ||||
if (d->bd_direction == BPF_D_INOUT) | if (d->bd_direction == BPF_D_INOUT) | ||||
m->m_flags |= M_PROMISC; | m->m_flags |= M_PROMISC; | ||||
} else | } else | ||||
mc = NULL; | mc = NULL; | ||||
m->m_pkthdr.len -= hlen; | m->m_pkthdr.len -= hlen; | ||||
m->m_len -= hlen; | m->m_len -= hlen; | ||||
m->m_data += hlen; /* XXX */ | m->m_data += hlen; /* XXX */ | ||||
CURVNET_SET(ifp->if_vnet); | CURVNET_SET(ifp->if_vnet); | ||||
#ifdef MAC | #ifdef MAC | ||||
BPFD_LOCK(d); | |||||
mac_bpfdesc_create_mbuf(d, m); | mac_bpfdesc_create_mbuf(d, m); | ||||
if (mc != NULL) | if (mc != NULL) | ||||
mac_bpfdesc_create_mbuf(d, mc); | mac_bpfdesc_create_mbuf(d, mc); | ||||
BPFD_UNLOCK(d); | |||||
#endif | #endif | ||||
bzero(&ro, sizeof(ro)); | bzero(&ro, sizeof(ro)); | ||||
if (hlen != 0) { | if (hlen != 0) { | ||||
ro.ro_prepend = (u_char *)&dst.sa_data; | ro.ro_prepend = (u_char *)&dst.sa_data; | ||||
ro.ro_plen = hlen; | ro.ro_plen = hlen; | ||||
ro.ro_flags = RT_HAS_HEADER; | ro.ro_flags = RT_HAS_HEADER; | ||||
} | } | ||||
error = (*ifp->if_output)(ifp, m, &dst, &ro); | error = (*ifp->if_output)(ifp, m, &dst, &ro); | ||||
if (error) | if (error) | ||||
counter_u64_add(d->bd_wdcount, 1); | counter_u64_add(d->bd_wdcount, 1); | ||||
if (mc != NULL) { | if (mc != NULL) { | ||||
if (error == 0) | if (error == 0) | ||||
(*ifp->if_input)(ifp, mc); | (*ifp->if_input)(ifp, mc); | ||||
else | else | ||||
m_freem(mc); | m_freem(mc); | ||||
} | } | ||||
CURVNET_RESTORE(); | CURVNET_RESTORE(); | ||||
BPFD_UNLOCK(d); | |||||
bpfd_rele(d); | |||||
return (error); | |||||
out_locked: | |||||
counter_u64_add(d->bd_wdcount, 1); | |||||
NET_EPOCH_EXIT(et); | |||||
BPFD_UNLOCK(d); | |||||
return (error); | return (error); | ||||
} | } | ||||
/* | /* | ||||
* Reset a descriptor by flushing its packet buffer and clearing the receive | * Reset a descriptor by flushing its packet buffer and clearing the receive | ||||
* and drop counts. This is doable for kernel-only buffers, but with | * and drop counts. This is doable for kernel-only buffers, but with | ||||
* zero-copy buffers, we can't write to (or rotate) buffers that are | * zero-copy buffers, we can't write to (or rotate) buffers that are | ||||
* currently owned by userspace. It would be nice if we could encapsulate | * currently owned by userspace. It would be nice if we could encapsulate | ||||
▲ Show 20 Lines • Show All 608 Lines • ▼ Show 20 Lines | case BIOCROTZBUF: | ||||
error = bpf_ioctl_rotzbuf(td, d, (struct bpf_zbuf *)addr); | error = bpf_ioctl_rotzbuf(td, d, (struct bpf_zbuf *)addr); | ||||
break; | break; | ||||
} | } | ||||
CURVNET_RESTORE(); | CURVNET_RESTORE(); | ||||
return (error); | return (error); | ||||
} | } | ||||
/* | /* | ||||
* Set d's packet filter program to fp. If this file already has a filter, | * Set d's packet filter program to fp. If this file already has a filter, | ||||
* free it and replace it. Returns EINVAL for bogus requests. | * free it and replace it. Returns EINVAL for bogus requests. | ||||
* | * | ||||
* Note we need global lock here to serialize bpf_setf() and bpf_setif() calls | * Note we use global lock here to serialize bpf_setf() and bpf_setif() | ||||
* since reading d->bd_bif can't be protected by d or interface lock due to | * calls. | ||||
* lock order. | |||||
* | |||||
* Additionally, we have to acquire interface write lock due to bpf_mtap() uses | |||||
* interface read lock to read all filers. | |||||
* | |||||
*/ | */ | ||||
static int | static int | ||||
bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd) | bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd) | ||||
{ | { | ||||
#ifdef COMPAT_FREEBSD32 | #ifdef COMPAT_FREEBSD32 | ||||
struct bpf_program fp_swab; | struct bpf_program fp_swab; | ||||
struct bpf_program32 *fp32; | struct bpf_program32 *fp32; | ||||
#endif | #endif | ||||
struct bpf_insn *fcode, *old; | struct bpf_program_buffer *fcode; | ||||
struct bpf_insn *filter; | |||||
#ifdef BPF_JITTER | #ifdef BPF_JITTER | ||||
bpf_jit_filter *jfunc, *ofunc; | bpf_jit_filter *jfunc; | ||||
#endif | #endif | ||||
size_t size; | size_t size; | ||||
u_int flen; | u_int flen; | ||||
int need_upgrade; | bool track_event; | ||||
#ifdef COMPAT_FREEBSD32 | #ifdef COMPAT_FREEBSD32 | ||||
switch (cmd) { | switch (cmd) { | ||||
case BIOCSETF32: | case BIOCSETF32: | ||||
case BIOCSETWF32: | case BIOCSETWF32: | ||||
case BIOCSETFNR32: | case BIOCSETFNR32: | ||||
fp32 = (struct bpf_program32 *)fp; | fp32 = (struct bpf_program32 *)fp; | ||||
fp_swab.bf_len = fp32->bf_len; | fp_swab.bf_len = fp32->bf_len; | ||||
fp_swab.bf_insns = (struct bpf_insn *)(uintptr_t)fp32->bf_insns; | fp_swab.bf_insns = | ||||
(struct bpf_insn *)(uintptr_t)fp32->bf_insns; | |||||
fp = &fp_swab; | fp = &fp_swab; | ||||
switch (cmd) { | switch (cmd) { | ||||
case BIOCSETF32: | case BIOCSETF32: | ||||
cmd = BIOCSETF; | cmd = BIOCSETF; | ||||
break; | break; | ||||
case BIOCSETWF32: | case BIOCSETWF32: | ||||
cmd = BIOCSETWF; | cmd = BIOCSETWF; | ||||
break; | break; | ||||
} | } | ||||
break; | break; | ||||
} | } | ||||
#endif | #endif | ||||
fcode = NULL; | filter = NULL; | ||||
#ifdef BPF_JITTER | #ifdef BPF_JITTER | ||||
jfunc = ofunc = NULL; | jfunc = NULL; | ||||
#endif | #endif | ||||
need_upgrade = 0; | |||||
/* | /* | ||||
* Check new filter validness before acquiring any locks. | * Check new filter validness before acquiring any locks. | ||||
* Allocate memory for new filter, if needed. | * Allocate memory for new filter, if needed. | ||||
*/ | */ | ||||
flen = fp->bf_len; | flen = fp->bf_len; | ||||
if (flen > bpf_maxinsns || (fp->bf_insns == NULL && flen != 0)) | if (flen > bpf_maxinsns || (fp->bf_insns == NULL && flen != 0)) | ||||
return (EINVAL); | return (EINVAL); | ||||
size = flen * sizeof(*fp->bf_insns); | size = flen * sizeof(*fp->bf_insns); | ||||
if (size > 0) { | if (size > 0) { | ||||
/* We're setting up new filter. Copy and check actual data. */ | /* We're setting up new filter. Copy and check actual data. */ | ||||
fcode = malloc(size, M_BPF, M_WAITOK); | fcode = bpf_program_buffer_alloc(size, M_WAITOK); | ||||
if (copyin(fp->bf_insns, fcode, size) != 0 || | filter = (struct bpf_insn *)fcode->buffer; | ||||
!bpf_validate(fcode, flen)) { | if (copyin(fp->bf_insns, filter, size) != 0 || | ||||
!bpf_validate(filter, flen)) { | |||||
free(fcode, M_BPF); | free(fcode, M_BPF); | ||||
return (EINVAL); | return (EINVAL); | ||||
} | } | ||||
#ifdef BPF_JITTER | #ifdef BPF_JITTER | ||||
if (cmd != BIOCSETWF) { | if (cmd != BIOCSETWF) { | ||||
/* | /* | ||||
* Filter is copied inside fcode and is | * Filter is copied inside fcode and is | ||||
* perfectly valid. | * perfectly valid. | ||||
*/ | */ | ||||
jfunc = bpf_jitter(fcode, flen); | jfunc = bpf_jitter(filter, flen); | ||||
} | } | ||||
#endif | #endif | ||||
} | } | ||||
BPF_LOCK(); | track_event = false; | ||||
fcode = NULL; | |||||
/* | BPF_LOCK(); | ||||
* Set up new filter. | |||||
* Protect filter change by interface lock. | |||||
* Additionally, we are protected by global lock here. | |||||
*/ | |||||
if (d->bd_bif != NULL) | |||||
BPFIF_WLOCK(d->bd_bif); | |||||
BPFD_LOCK(d); | BPFD_LOCK(d); | ||||
/* Set up new filter. */ | |||||
if (cmd == BIOCSETWF) { | if (cmd == BIOCSETWF) { | ||||
old = d->bd_wfilter; | if (d->bd_wfilter != NULL) { | ||||
d->bd_wfilter = fcode; | fcode = __containerof((void *)d->bd_wfilter, | ||||
struct bpf_program_buffer, buffer); | |||||
#ifdef BPF_JITTER | |||||
fcode->func = NULL; | |||||
#endif | |||||
} | |||||
d->bd_wfilter = filter; | |||||
} else { | } else { | ||||
old = d->bd_rfilter; | if (d->bd_rfilter != NULL) { | ||||
d->bd_rfilter = fcode; | fcode = __containerof((void *)d->bd_rfilter, | ||||
struct bpf_program_buffer, buffer); | |||||
#ifdef BPF_JITTER | #ifdef BPF_JITTER | ||||
ofunc = d->bd_bfilter; | fcode->func = d->bd_bfilter; | ||||
#endif | |||||
} | |||||
d->bd_rfilter = filter; | |||||
#ifdef BPF_JITTER | |||||
d->bd_bfilter = jfunc; | d->bd_bfilter = jfunc; | ||||
#endif | #endif | ||||
if (cmd == BIOCSETF) | if (cmd == BIOCSETF) | ||||
reset_d(d); | reset_d(d); | ||||
need_upgrade = bpf_check_upgrade(cmd, d, fcode, flen); | if (bpf_check_upgrade(cmd, d, filter, flen) != 0) { | ||||
/* | |||||
* Filter can be set several times without | |||||
* specifying interface. In this case just mark d | |||||
* as reader. | |||||
*/ | |||||
d->bd_writer = 0; | |||||
if (d->bd_bif != NULL) { | |||||
/* | |||||
* Remove descriptor from writers-only list | |||||
* and add it to active readers list. | |||||
*/ | |||||
CK_LIST_REMOVE(d, bd_next); | |||||
CK_LIST_INSERT_HEAD(&d->bd_bif->bif_dlist, | |||||
d, bd_next); | |||||
CTR2(KTR_NET, | |||||
"%s: upgrade required by pid %d", | |||||
__func__, d->bd_pid); | |||||
track_event = true; | |||||
} | } | ||||
} | |||||
} | |||||
BPFD_UNLOCK(d); | BPFD_UNLOCK(d); | ||||
if (d->bd_bif != NULL) | |||||
BPFIF_WUNLOCK(d->bd_bif); | |||||
if (old != NULL) | |||||
free(old, M_BPF); | |||||
#ifdef BPF_JITTER | |||||
if (ofunc != NULL) | |||||
bpf_destroy_jit_filter(ofunc); | |||||
#endif | |||||
/* Move d to active readers list. */ | if (fcode != NULL) | ||||
if (need_upgrade != 0) | epoch_call(net_epoch_preempt, &fcode->epoch_ctx, | ||||
bpf_upgraded(d); | bpf_program_buffer_free); | ||||
if (track_event) | |||||
EVENTHANDLER_INVOKE(bpf_track, | |||||
d->bd_bif->bif_ifp, d->bd_bif->bif_dlt, 1); | |||||
BPF_UNLOCK(); | BPF_UNLOCK(); | ||||
return (0); | return (0); | ||||
} | } | ||||
/* | /* | ||||
* Detach a file from its current interface (if attached at all) and attach | * Detach a file from its current interface (if attached at all) and attach | ||||
* to the interface indicated by the name stored in ifr. | * to the interface indicated by the name stored in ifr. | ||||
* Return an errno or 0. | * Return an errno or 0. | ||||
*/ | */ | ||||
static int | static int | ||||
bpf_setif(struct bpf_d *d, struct ifreq *ifr) | bpf_setif(struct bpf_d *d, struct ifreq *ifr) | ||||
{ | { | ||||
struct bpf_if *bp; | struct bpf_if *bp; | ||||
struct ifnet *theywant; | struct ifnet *theywant; | ||||
BPF_LOCK_ASSERT(); | BPF_LOCK_ASSERT(); | ||||
theywant = ifunit(ifr->ifr_name); | theywant = ifunit(ifr->ifr_name); | ||||
if (theywant == NULL || theywant->if_bpf == NULL) | if (theywant == NULL || theywant->if_bpf == NULL) | ||||
return (ENXIO); | return (ENXIO); | ||||
bp = theywant->if_bpf; | bp = theywant->if_bpf; | ||||
/* Check if interface is not being detached from BPF */ | |||||
BPFIF_RLOCK(bp); | |||||
if (bp->bif_flags & BPFIF_FLAG_DYING) { | |||||
BPFIF_RUNLOCK(bp); | |||||
return (ENXIO); | |||||
} | |||||
BPFIF_RUNLOCK(bp); | |||||
/* | /* | ||||
* At this point, we expect the buffer is already allocated. If not, | * At this point, we expect the buffer is already allocated. If not, | ||||
* return an error. | * return an error. | ||||
*/ | */ | ||||
switch (d->bd_bufmode) { | switch (d->bd_bufmode) { | ||||
case BPF_BUFMODE_BUFFER: | case BPF_BUFMODE_BUFFER: | ||||
case BPF_BUFMODE_ZBUF: | case BPF_BUFMODE_ZBUF: | ||||
if (d->bd_sbuf == NULL) | if (d->bd_sbuf == NULL) | ||||
return (EINVAL); | return (EINVAL); | ||||
break; | break; | ||||
default: | default: | ||||
panic("bpf_setif: bufmode %d", d->bd_bufmode); | panic("bpf_setif: bufmode %d", d->bd_bufmode); | ||||
} | } | ||||
if (bp != d->bd_bif) | if (bp != d->bd_bif) | ||||
bpf_attachd(d, bp); | bpf_attachd(d, bp); | ||||
else { | |||||
BPFD_LOCK(d); | BPFD_LOCK(d); | ||||
reset_d(d); | reset_d(d); | ||||
BPFD_UNLOCK(d); | BPFD_UNLOCK(d); | ||||
} | |||||
return (0); | return (0); | ||||
} | } | ||||
/* | /* | ||||
* Support for select() and poll() system calls | * Support for select() and poll() system calls | ||||
* | * | ||||
* Return true iff the specific operation will not block indefinitely. | * Return true iff the specific operation will not block indefinitely. | ||||
* Otherwise, return false but make a note that a selwakeup() must be done. | * Otherwise, return false but make a note that a selwakeup() must be done. | ||||
▲ Show 20 Lines • Show All 135 Lines • ▼ Show 20 Lines | |||||
* Incoming linkage from device drivers. Process the packet pkt, of length | * Incoming linkage from device drivers. Process the packet pkt, of length | ||||
* pktlen, which is stored in a contiguous buffer. The packet is parsed | * pktlen, which is stored in a contiguous buffer. The packet is parsed | ||||
* by each process' filter, and if accepted, stashed into the corresponding | * by each process' filter, and if accepted, stashed into the corresponding | ||||
* buffer. | * buffer. | ||||
*/ | */ | ||||
void | void | ||||
bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) | bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) | ||||
{ | { | ||||
struct epoch_tracker et; | |||||
struct bintime bt; | struct bintime bt; | ||||
struct bpf_d *d; | struct bpf_d *d; | ||||
#ifdef BPF_JITTER | #ifdef BPF_JITTER | ||||
bpf_jit_filter *bf; | bpf_jit_filter *bf; | ||||
#endif | #endif | ||||
u_int slen; | u_int slen; | ||||
int gottime; | int gottime; | ||||
gottime = BPF_TSTAMP_NONE; | gottime = BPF_TSTAMP_NONE; | ||||
NET_EPOCH_ENTER(et); | |||||
BPFIF_RLOCK(bp); | CK_LIST_FOREACH(d, &bp->bif_dlist, bd_next) { | ||||
LIST_FOREACH(d, &bp->bif_dlist, bd_next) { | |||||
/* | |||||
* We are not using any locks for d here because: | |||||
* 1) any filter change is protected by interface | |||||
* write lock | |||||
* 2) destroying/detaching d is protected by interface | |||||
* write lock, too | |||||
*/ | |||||
counter_u64_add(d->bd_rcount, 1); | counter_u64_add(d->bd_rcount, 1); | ||||
/* | /* | ||||
* NB: We dont call BPF_CHECK_DIRECTION() here since there is no | * NB: We dont call BPF_CHECK_DIRECTION() here since there | ||||
* way for the caller to indiciate to us whether this packet | * is no way for the caller to indiciate to us whether this | ||||
* is inbound or outbound. In the bpf_mtap() routines, we use | * packet is inbound or outbound. In the bpf_mtap() routines, | ||||
* the interface pointers on the mbuf to figure it out. | * we use the interface pointers on the mbuf to figure it out. | ||||
*/ | */ | ||||
#ifdef BPF_JITTER | #ifdef BPF_JITTER | ||||
bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL; | bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL; | ||||
if (bf != NULL) | if (bf != NULL) | ||||
slen = (*(bf->func))(pkt, pktlen, pktlen); | slen = (*(bf->func))(pkt, pktlen, pktlen); | ||||
else | else | ||||
#endif | #endif | ||||
slen = bpf_filter(d->bd_rfilter, pkt, pktlen, pktlen); | slen = bpf_filter(d->bd_rfilter, pkt, pktlen, pktlen); | ||||
if (slen != 0) { | if (slen != 0) { | ||||
/* | /* | ||||
* Filter matches. Let's to acquire write lock. | * Filter matches. Let's to acquire write lock. | ||||
*/ | */ | ||||
BPFD_LOCK(d); | BPFD_LOCK(d); | ||||
counter_u64_add(d->bd_fcount, 1); | counter_u64_add(d->bd_fcount, 1); | ||||
if (gottime < bpf_ts_quality(d->bd_tstamp)) | if (gottime < bpf_ts_quality(d->bd_tstamp)) | ||||
gottime = bpf_gettime(&bt, d->bd_tstamp, NULL); | gottime = bpf_gettime(&bt, d->bd_tstamp, | ||||
NULL); | |||||
#ifdef MAC | #ifdef MAC | ||||
if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) | if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) | ||||
#endif | #endif | ||||
catchpacket(d, pkt, pktlen, slen, | catchpacket(d, pkt, pktlen, slen, | ||||
bpf_append_bytes, &bt); | bpf_append_bytes, &bt); | ||||
BPFD_UNLOCK(d); | BPFD_UNLOCK(d); | ||||
} | } | ||||
} | } | ||||
BPFIF_RUNLOCK(bp); | NET_EPOCH_EXIT(et); | ||||
} | } | ||||
#define BPF_CHECK_DIRECTION(d, r, i) \ | #define BPF_CHECK_DIRECTION(d, r, i) \ | ||||
(((d)->bd_direction == BPF_D_IN && (r) != (i)) || \ | (((d)->bd_direction == BPF_D_IN && (r) != (i)) || \ | ||||
((d)->bd_direction == BPF_D_OUT && (r) == (i))) | ((d)->bd_direction == BPF_D_OUT && (r) == (i))) | ||||
/* | /* | ||||
* Incoming linkage from device drivers, when packet is in an mbuf chain. | * Incoming linkage from device drivers, when packet is in an mbuf chain. | ||||
* Locking model is explained in bpf_tap(). | * Locking model is explained in bpf_tap(). | ||||
*/ | */ | ||||
void | void | ||||
bpf_mtap(struct bpf_if *bp, struct mbuf *m) | bpf_mtap(struct bpf_if *bp, struct mbuf *m) | ||||
{ | { | ||||
struct epoch_tracker et; | |||||
struct bintime bt; | struct bintime bt; | ||||
struct bpf_d *d; | struct bpf_d *d; | ||||
#ifdef BPF_JITTER | #ifdef BPF_JITTER | ||||
bpf_jit_filter *bf; | bpf_jit_filter *bf; | ||||
#endif | #endif | ||||
u_int pktlen, slen; | u_int pktlen, slen; | ||||
int gottime; | int gottime; | ||||
/* Skip outgoing duplicate packets. */ | /* Skip outgoing duplicate packets. */ | ||||
if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { | if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { | ||||
m->m_flags &= ~M_PROMISC; | m->m_flags &= ~M_PROMISC; | ||||
return; | return; | ||||
} | } | ||||
pktlen = m_length(m, NULL); | pktlen = m_length(m, NULL); | ||||
gottime = BPF_TSTAMP_NONE; | gottime = BPF_TSTAMP_NONE; | ||||
BPFIF_RLOCK(bp); | NET_EPOCH_ENTER(et); | ||||
CK_LIST_FOREACH(d, &bp->bif_dlist, bd_next) { | |||||
LIST_FOREACH(d, &bp->bif_dlist, bd_next) { | |||||
if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp)) | if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp)) | ||||
continue; | continue; | ||||
counter_u64_add(d->bd_rcount, 1); | counter_u64_add(d->bd_rcount, 1); | ||||
#ifdef BPF_JITTER | #ifdef BPF_JITTER | ||||
bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL; | bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL; | ||||
/* XXX We cannot handle multiple mbufs. */ | /* XXX We cannot handle multiple mbufs. */ | ||||
if (bf != NULL && m->m_next == NULL) | if (bf != NULL && m->m_next == NULL) | ||||
slen = (*(bf->func))(mtod(m, u_char *), pktlen, pktlen); | slen = (*(bf->func))(mtod(m, u_char *), pktlen, | ||||
pktlen); | |||||
else | else | ||||
#endif | #endif | ||||
slen = bpf_filter(d->bd_rfilter, (u_char *)m, pktlen, 0); | slen = bpf_filter(d->bd_rfilter, (u_char *)m, pktlen, 0); | ||||
if (slen != 0) { | if (slen != 0) { | ||||
BPFD_LOCK(d); | BPFD_LOCK(d); | ||||
counter_u64_add(d->bd_fcount, 1); | counter_u64_add(d->bd_fcount, 1); | ||||
if (gottime < bpf_ts_quality(d->bd_tstamp)) | if (gottime < bpf_ts_quality(d->bd_tstamp)) | ||||
gottime = bpf_gettime(&bt, d->bd_tstamp, m); | gottime = bpf_gettime(&bt, d->bd_tstamp, m); | ||||
#ifdef MAC | #ifdef MAC | ||||
if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) | if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) | ||||
#endif | #endif | ||||
catchpacket(d, (u_char *)m, pktlen, slen, | catchpacket(d, (u_char *)m, pktlen, slen, | ||||
bpf_append_mbuf, &bt); | bpf_append_mbuf, &bt); | ||||
BPFD_UNLOCK(d); | BPFD_UNLOCK(d); | ||||
} | } | ||||
} | } | ||||
BPFIF_RUNLOCK(bp); | NET_EPOCH_EXIT(et); | ||||
} | } | ||||
/* | /* | ||||
* Incoming linkage from device drivers, when packet is in | * Incoming linkage from device drivers, when packet is in | ||||
* an mbuf chain and to be prepended by a contiguous header. | * an mbuf chain and to be prepended by a contiguous header. | ||||
*/ | */ | ||||
void | void | ||||
bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m) | bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m) | ||||
{ | { | ||||
struct epoch_tracker et; | |||||
struct bintime bt; | struct bintime bt; | ||||
struct mbuf mb; | struct mbuf mb; | ||||
struct bpf_d *d; | struct bpf_d *d; | ||||
u_int pktlen, slen; | u_int pktlen, slen; | ||||
int gottime; | int gottime; | ||||
/* Skip outgoing duplicate packets. */ | /* Skip outgoing duplicate packets. */ | ||||
if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { | if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { | ||||
Show All 9 Lines | bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m) | ||||
*/ | */ | ||||
mb.m_next = m; | mb.m_next = m; | ||||
mb.m_data = data; | mb.m_data = data; | ||||
mb.m_len = dlen; | mb.m_len = dlen; | ||||
pktlen += dlen; | pktlen += dlen; | ||||
gottime = BPF_TSTAMP_NONE; | gottime = BPF_TSTAMP_NONE; | ||||
BPFIF_RLOCK(bp); | NET_EPOCH_ENTER(et); | ||||
CK_LIST_FOREACH(d, &bp->bif_dlist, bd_next) { | |||||
LIST_FOREACH(d, &bp->bif_dlist, bd_next) { | |||||
if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp)) | if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp)) | ||||
continue; | continue; | ||||
counter_u64_add(d->bd_rcount, 1); | counter_u64_add(d->bd_rcount, 1); | ||||
slen = bpf_filter(d->bd_rfilter, (u_char *)&mb, pktlen, 0); | slen = bpf_filter(d->bd_rfilter, (u_char *)&mb, pktlen, 0); | ||||
if (slen != 0) { | if (slen != 0) { | ||||
BPFD_LOCK(d); | BPFD_LOCK(d); | ||||
counter_u64_add(d->bd_fcount, 1); | counter_u64_add(d->bd_fcount, 1); | ||||
if (gottime < bpf_ts_quality(d->bd_tstamp)) | if (gottime < bpf_ts_quality(d->bd_tstamp)) | ||||
gottime = bpf_gettime(&bt, d->bd_tstamp, m); | gottime = bpf_gettime(&bt, d->bd_tstamp, m); | ||||
#ifdef MAC | #ifdef MAC | ||||
if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) | if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) | ||||
#endif | #endif | ||||
catchpacket(d, (u_char *)&mb, pktlen, slen, | catchpacket(d, (u_char *)&mb, pktlen, slen, | ||||
bpf_append_mbuf, &bt); | bpf_append_mbuf, &bt); | ||||
BPFD_UNLOCK(d); | BPFD_UNLOCK(d); | ||||
} | } | ||||
} | } | ||||
BPFIF_RUNLOCK(bp); | NET_EPOCH_EXIT(et); | ||||
} | } | ||||
#undef BPF_CHECK_DIRECTION | #undef BPF_CHECK_DIRECTION | ||||
#undef BPF_TSTAMP_NONE | #undef BPF_TSTAMP_NONE | ||||
#undef BPF_TSTAMP_FAST | #undef BPF_TSTAMP_FAST | ||||
#undef BPF_TSTAMP_NORMAL | #undef BPF_TSTAMP_NORMAL | ||||
#undef BPF_TSTAMP_EXTERN | #undef BPF_TSTAMP_EXTERN | ||||
static int | static int | ||||
bpf_hdrlen(struct bpf_d *d) | bpf_hdrlen(struct bpf_d *d) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 202 Lines • ▼ Show 20 Lines | if (do_wakeup) | ||||
bpf_wakeup(d); | bpf_wakeup(d); | ||||
} | } | ||||
/* | /* | ||||
* Free buffers currently in use by a descriptor. | * Free buffers currently in use by a descriptor. | ||||
* Called on close. | * Called on close. | ||||
*/ | */ | ||||
static void | static void | ||||
bpf_freed(struct bpf_d *d) | bpfd_free(epoch_context_t ctx) | ||||
{ | { | ||||
struct bpf_d *d; | |||||
struct bpf_program_buffer *p; | |||||
/* | /* | ||||
* We don't need to lock out interrupts since this descriptor has | * We don't need to lock out interrupts since this descriptor has | ||||
* been detached from its interface and it yet hasn't been marked | * been detached from its interface and it yet hasn't been marked | ||||
* free. | * free. | ||||
*/ | */ | ||||
d = __containerof(ctx, struct bpf_d, epoch_ctx); | |||||
bpf_free(d); | bpf_free(d); | ||||
if (d->bd_rfilter != NULL) { | if (d->bd_rfilter != NULL) { | ||||
free((caddr_t)d->bd_rfilter, M_BPF); | p = __containerof((void *)d->bd_rfilter, | ||||
#ifdef BPF_JITTER | struct bpf_program_buffer, buffer); | ||||
if (d->bd_bfilter != NULL) | bpf_program_buffer_free(&p->epoch_ctx); | ||||
bpf_destroy_jit_filter(d->bd_bfilter); | |||||
#endif | |||||
} | } | ||||
if (d->bd_wfilter != NULL) | if (d->bd_wfilter != NULL) { | ||||
free((caddr_t)d->bd_wfilter, M_BPF); | p = __containerof((void *)d->bd_wfilter, | ||||
mtx_destroy(&d->bd_lock); | struct bpf_program_buffer, buffer); | ||||
bpf_program_buffer_free(&p->epoch_ctx); | |||||
} | |||||
mtx_destroy(&d->bd_lock); | |||||
counter_u64_free(d->bd_rcount); | counter_u64_free(d->bd_rcount); | ||||
counter_u64_free(d->bd_dcount); | counter_u64_free(d->bd_dcount); | ||||
counter_u64_free(d->bd_fcount); | counter_u64_free(d->bd_fcount); | ||||
counter_u64_free(d->bd_wcount); | counter_u64_free(d->bd_wcount); | ||||
counter_u64_free(d->bd_wfcount); | counter_u64_free(d->bd_wfcount); | ||||
counter_u64_free(d->bd_wdcount); | counter_u64_free(d->bd_wdcount); | ||||
counter_u64_free(d->bd_zcopy); | counter_u64_free(d->bd_zcopy); | ||||
free(d, M_BPF); | |||||
} | } | ||||
/* | /* | ||||
* Attach an interface to bpf. dlt is the link layer type; hdrlen is the | * Attach an interface to bpf. dlt is the link layer type; hdrlen is the | ||||
* fixed size of the link header (variable length headers not yet supported). | * fixed size of the link header (variable length headers not yet supported). | ||||
*/ | */ | ||||
void | void | ||||
bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen) | bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen) | ||||
{ | { | ||||
bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf); | bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf); | ||||
} | } | ||||
/* | /* | ||||
* Attach an interface to bpf. ifp is a pointer to the structure | * Attach an interface to bpf. ifp is a pointer to the structure | ||||
* defining the interface to be attached, dlt is the link layer type, | * defining the interface to be attached, dlt is the link layer type, | ||||
* and hdrlen is the fixed size of the link header (variable length | * and hdrlen is the fixed size of the link header (variable length | ||||
* headers are not yet supporrted). | * headers are not yet supporrted). | ||||
*/ | */ | ||||
void | void | ||||
bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) | bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, | ||||
struct bpf_if **driverp) | |||||
{ | { | ||||
struct bpf_if *bp; | struct bpf_if *bp; | ||||
KASSERT(*driverp == NULL, ("bpfattach2: driverp already initialized")); | KASSERT(*driverp == NULL, | ||||
("bpfattach2: driverp already initialized")); | |||||
bp = malloc(sizeof(*bp), M_BPF, M_WAITOK | M_ZERO); | bp = malloc(sizeof(*bp), M_BPF, M_WAITOK | M_ZERO); | ||||
rw_init(&bp->bif_lock, "bpf interface lock"); | CK_LIST_INIT(&bp->bif_dlist); | ||||
LIST_INIT(&bp->bif_dlist); | CK_LIST_INIT(&bp->bif_wlist); | ||||
LIST_INIT(&bp->bif_wlist); | |||||
bp->bif_ifp = ifp; | bp->bif_ifp = ifp; | ||||
bp->bif_dlt = dlt; | bp->bif_dlt = dlt; | ||||
bp->bif_hdrlen = hdrlen; | bp->bif_hdrlen = hdrlen; | ||||
bp->bif_bpf = driverp; | bp->bif_bpf = driverp; | ||||
bp->bif_refcnt = 1; | |||||
*driverp = bp; | *driverp = bp; | ||||
/* | |||||
* Reference ifnet pointer, so it won't freed until | |||||
* we release it. | |||||
*/ | |||||
if_ref(ifp); | |||||
BPF_LOCK(); | BPF_LOCK(); | ||||
LIST_INSERT_HEAD(&bpf_iflist, bp, bif_next); | CK_LIST_INSERT_HEAD(&bpf_iflist, bp, bif_next); | ||||
BPF_UNLOCK(); | BPF_UNLOCK(); | ||||
if (bootverbose && IS_DEFAULT_VNET(curvnet)) | if (bootverbose && IS_DEFAULT_VNET(curvnet)) | ||||
if_printf(ifp, "bpf attached\n"); | if_printf(ifp, "bpf attached\n"); | ||||
} | } | ||||
#ifdef VIMAGE | #ifdef VIMAGE | ||||
/* | /* | ||||
Show All 24 Lines | |||||
/* | /* | ||||
* Detach bpf from an interface. This involves detaching each descriptor | * Detach bpf from an interface. This involves detaching each descriptor | ||||
* associated with the interface. Notify each descriptor as it's detached | * associated with the interface. Notify each descriptor as it's detached | ||||
* so that any sleepers wake up and get ENXIO. | * so that any sleepers wake up and get ENXIO. | ||||
*/ | */ | ||||
void | void | ||||
bpfdetach(struct ifnet *ifp) | bpfdetach(struct ifnet *ifp) | ||||
{ | { | ||||
struct bpf_if *bp, *bp_temp; | struct bpf_if *bp, *bp_temp; | ||||
struct bpf_d *d; | struct bpf_d *d; | ||||
int ndetached; | |||||
ndetached = 0; | |||||
BPF_LOCK(); | BPF_LOCK(); | ||||
/* Find all bpf_if struct's which reference ifp and detach them. */ | /* Find all bpf_if struct's which reference ifp and detach them. */ | ||||
LIST_FOREACH_SAFE(bp, &bpf_iflist, bif_next, bp_temp) { | CK_LIST_FOREACH_SAFE(bp, &bpf_iflist, bif_next, bp_temp) { | ||||
if (ifp != bp->bif_ifp) | if (ifp != bp->bif_ifp) | ||||
continue; | continue; | ||||
LIST_REMOVE(bp, bif_next); | CK_LIST_REMOVE(bp, bif_next); | ||||
/* Add to to-be-freed list */ | |||||
LIST_INSERT_HEAD(&bpf_freelist, bp, bif_next); | |||||
ndetached++; | |||||
/* | |||||
* Delay freeing bp till interface is detached | |||||
* and all routes through this interface are removed. | |||||
* Mark bp as detached to restrict new consumers. | |||||
*/ | |||||
BPFIF_WLOCK(bp); | |||||
bp->bif_flags |= BPFIF_FLAG_DYING; | |||||
*bp->bif_bpf = (struct bpf_if *)&dead_bpf_if; | *bp->bif_bpf = (struct bpf_if *)&dead_bpf_if; | ||||
BPFIF_WUNLOCK(bp); | |||||
CTR4(KTR_NET, "%s: sheduling free for encap %d (%p) for if %p", | CTR4(KTR_NET, | ||||
"%s: sheduling free for encap %d (%p) for if %p", | |||||
__func__, bp->bif_dlt, bp, ifp); | __func__, bp->bif_dlt, bp, ifp); | ||||
/* Free common descriptors */ | /* Detach common descriptors */ | ||||
while ((d = LIST_FIRST(&bp->bif_dlist)) != NULL) { | while ((d = CK_LIST_FIRST(&bp->bif_dlist)) != NULL) { | ||||
bpf_detachd_locked(d); | bpf_detachd_locked(d, true); | ||||
BPFD_LOCK(d); | |||||
bpf_wakeup(d); | |||||
BPFD_UNLOCK(d); | |||||
} | } | ||||
/* Free writer-only descriptors */ | /* Detach writer-only descriptors */ | ||||
while ((d = LIST_FIRST(&bp->bif_wlist)) != NULL) { | while ((d = CK_LIST_FIRST(&bp->bif_wlist)) != NULL) { | ||||
bpf_detachd_locked(d); | bpf_detachd_locked(d, true); | ||||
BPFD_LOCK(d); | |||||
bpf_wakeup(d); | |||||
BPFD_UNLOCK(d); | |||||
} | } | ||||
bpfif_rele(bp); | |||||
} | } | ||||
BPF_UNLOCK(); | BPF_UNLOCK(); | ||||
#ifdef INVARIANTS | |||||
if (ndetached == 0) | |||||
printf("bpfdetach: %s was not attached\n", ifp->if_xname); | |||||
#endif | |||||
} | } | ||||
/* | /* | ||||
* Interface departure handler. | |||||
* Note departure event does not guarantee interface is going down. | |||||
* Interface renaming is currently done via departure/arrival event set. | |||||
* | |||||
* Departure handled is called after all routes pointing to | |||||
* given interface are removed and interface is in down state | |||||
* restricting any packets to be sent/received. We assume it is now safe | |||||
* to free data allocated by BPF. | |||||
*/ | |||||
static void | |||||
bpf_ifdetach(void *arg __unused, struct ifnet *ifp) | |||||
{ | |||||
struct bpf_if *bp, *bp_temp; | |||||
int nmatched = 0; | |||||
/* Ignore ifnet renaming. */ | |||||
if (ifp->if_flags & IFF_RENAMING) | |||||
return; | |||||
BPF_LOCK(); | |||||
/* | |||||
* Find matching entries in free list. | |||||
* Nothing should be found if bpfdetach() was not called. | |||||
*/ | |||||
LIST_FOREACH_SAFE(bp, &bpf_freelist, bif_next, bp_temp) { | |||||
if (ifp != bp->bif_ifp) | |||||
continue; | |||||
CTR3(KTR_NET, "%s: freeing BPF instance %p for interface %p", | |||||
__func__, bp, ifp); | |||||
LIST_REMOVE(bp, bif_next); | |||||
rw_destroy(&bp->bif_lock); | |||||
free(bp, M_BPF); | |||||
nmatched++; | |||||
} | |||||
BPF_UNLOCK(); | |||||
} | |||||
/* | |||||
* Get a list of available data link type of the interface. | * Get a list of available data link type of the interface. | ||||
*/ | */ | ||||
static int | static int | ||||
bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) | bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) | ||||
{ | { | ||||
struct ifnet *ifp; | struct ifnet *ifp; | ||||
struct bpf_if *bp; | struct bpf_if *bp; | ||||
u_int *lst; | u_int *lst; | ||||
int error, n, n1; | int error, n, n1; | ||||
BPF_LOCK_ASSERT(); | BPF_LOCK_ASSERT(); | ||||
ifp = d->bd_bif->bif_ifp; | ifp = d->bd_bif->bif_ifp; | ||||
again: | |||||
n1 = 0; | n1 = 0; | ||||
LIST_FOREACH(bp, &bpf_iflist, bif_next) { | CK_LIST_FOREACH(bp, &bpf_iflist, bif_next) { | ||||
if (bp->bif_ifp == ifp) | if (bp->bif_ifp == ifp) | ||||
n1++; | n1++; | ||||
} | } | ||||
if (bfl->bfl_list == NULL) { | if (bfl->bfl_list == NULL) { | ||||
bfl->bfl_len = n1; | bfl->bfl_len = n1; | ||||
return (0); | return (0); | ||||
} | } | ||||
if (n1 > bfl->bfl_len) | if (n1 > bfl->bfl_len) | ||||
return (ENOMEM); | return (ENOMEM); | ||||
BPF_UNLOCK(); | |||||
lst = malloc(n1 * sizeof(u_int), M_TEMP, M_WAITOK); | lst = malloc(n1 * sizeof(u_int), M_TEMP, M_WAITOK); | ||||
n = 0; | n = 0; | ||||
BPF_LOCK(); | CK_LIST_FOREACH(bp, &bpf_iflist, bif_next) { | ||||
LIST_FOREACH(bp, &bpf_iflist, bif_next) { | |||||
if (bp->bif_ifp != ifp) | if (bp->bif_ifp != ifp) | ||||
continue; | continue; | ||||
if (n >= n1) { | lst[n++] = bp->bif_dlt; | ||||
free(lst, M_TEMP); | |||||
goto again; | |||||
} | } | ||||
lst[n] = bp->bif_dlt; | |||||
n++; | |||||
} | |||||
BPF_UNLOCK(); | |||||
error = copyout(lst, bfl->bfl_list, sizeof(u_int) * n); | error = copyout(lst, bfl->bfl_list, sizeof(u_int) * n); | ||||
free(lst, M_TEMP); | free(lst, M_TEMP); | ||||
BPF_LOCK(); | |||||
bfl->bfl_len = n; | bfl->bfl_len = n; | ||||
return (error); | return (error); | ||||
} | } | ||||
/* | /* | ||||
* Set the data link type of a BPF instance. | * Set the data link type of a BPF instance. | ||||
*/ | */ | ||||
static int | static int | ||||
bpf_setdlt(struct bpf_d *d, u_int dlt) | bpf_setdlt(struct bpf_d *d, u_int dlt) | ||||
{ | { | ||||
int error, opromisc; | int error, opromisc; | ||||
struct ifnet *ifp; | struct ifnet *ifp; | ||||
struct bpf_if *bp; | struct bpf_if *bp; | ||||
BPF_LOCK_ASSERT(); | BPF_LOCK_ASSERT(); | ||||
MPASS(d->bd_bif != NULL); | |||||
/* | |||||
* It is safe to check bd_bif without BPFD_LOCK, it can not be | |||||
* changed while we hold global lock. | |||||
*/ | |||||
if (d->bd_bif->bif_dlt == dlt) | if (d->bd_bif->bif_dlt == dlt) | ||||
return (0); | return (0); | ||||
ifp = d->bd_bif->bif_ifp; | |||||
LIST_FOREACH(bp, &bpf_iflist, bif_next) { | ifp = d->bd_bif->bif_ifp; | ||||
CK_LIST_FOREACH(bp, &bpf_iflist, bif_next) { | |||||
if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) | if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) | ||||
break; | break; | ||||
} | } | ||||
if (bp == NULL) | |||||
return (EINVAL); | |||||
if (bp != NULL) { | |||||
opromisc = d->bd_promisc; | opromisc = d->bd_promisc; | ||||
bpf_attachd(d, bp); | bpf_attachd(d, bp); | ||||
BPFD_LOCK(d); | |||||
reset_d(d); | |||||
BPFD_UNLOCK(d); | |||||
if (opromisc) { | if (opromisc) { | ||||
error = ifpromisc(bp->bif_ifp, 1); | error = ifpromisc(bp->bif_ifp, 1); | ||||
if (error) | if (error) | ||||
if_printf(bp->bif_ifp, | if_printf(bp->bif_ifp, "%s: ifpromisc failed (%d)\n", | ||||
"bpf_setdlt: ifpromisc failed (%d)\n", | __func__, error); | ||||
error); | |||||
else | else | ||||
d->bd_promisc = 1; | d->bd_promisc = 1; | ||||
} | } | ||||
return (0); | |||||
} | } | ||||
return (bp == NULL ? EINVAL : 0); | |||||
} | |||||
static void | static void | ||||
bpf_drvinit(void *unused) | bpf_drvinit(void *unused) | ||||
{ | { | ||||
struct cdev *dev; | struct cdev *dev; | ||||
sx_init(&bpf_sx, "bpf global lock"); | sx_init(&bpf_sx, "bpf global lock"); | ||||
LIST_INIT(&bpf_iflist); | CK_LIST_INIT(&bpf_iflist); | ||||
LIST_INIT(&bpf_freelist); | |||||
dev = make_dev(&bpf_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "bpf"); | dev = make_dev(&bpf_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "bpf"); | ||||
/* For compatibility */ | /* For compatibility */ | ||||
make_dev_alias(dev, "bpf0"); | make_dev_alias(dev, "bpf0"); | ||||
/* Register interface departure handler */ | |||||
bpf_ifdetach_cookie = EVENTHANDLER_REGISTER( | |||||
ifnet_departure_event, bpf_ifdetach, NULL, | |||||
EVENTHANDLER_PRI_ANY); | |||||
} | } | ||||
/* | /* | ||||
* Zero out the various packet counters associated with all of the bpf | * Zero out the various packet counters associated with all of the bpf | ||||
* descriptors. At some point, we will probably want to get a bit more | * descriptors. At some point, we will probably want to get a bit more | ||||
* granular and allow the user to specify descriptors to be zeroed. | * granular and allow the user to specify descriptors to be zeroed. | ||||
*/ | */ | ||||
static void | static void | ||||
bpf_zero_counters(void) | bpf_zero_counters(void) | ||||
{ | { | ||||
struct bpf_if *bp; | struct bpf_if *bp; | ||||
struct bpf_d *bd; | struct bpf_d *bd; | ||||
BPF_LOCK(); | BPF_LOCK(); | ||||
LIST_FOREACH(bp, &bpf_iflist, bif_next) { | /* | ||||
BPFIF_RLOCK(bp); | * We are protected by global lock here, interfaces and | ||||
LIST_FOREACH(bd, &bp->bif_dlist, bd_next) { | * descriptors can not be deleted while we hold it. | ||||
BPFD_LOCK(bd); | */ | ||||
CK_LIST_FOREACH(bp, &bpf_iflist, bif_next) { | |||||
CK_LIST_FOREACH(bd, &bp->bif_dlist, bd_next) { | |||||
counter_u64_zero(bd->bd_rcount); | counter_u64_zero(bd->bd_rcount); | ||||
counter_u64_zero(bd->bd_dcount); | counter_u64_zero(bd->bd_dcount); | ||||
counter_u64_zero(bd->bd_fcount); | counter_u64_zero(bd->bd_fcount); | ||||
counter_u64_zero(bd->bd_wcount); | counter_u64_zero(bd->bd_wcount); | ||||
counter_u64_zero(bd->bd_wfcount); | counter_u64_zero(bd->bd_wfcount); | ||||
counter_u64_zero(bd->bd_zcopy); | counter_u64_zero(bd->bd_zcopy); | ||||
BPFD_UNLOCK(bd); | |||||
} | } | ||||
BPFIF_RUNLOCK(bp); | |||||
} | } | ||||
BPF_UNLOCK(); | BPF_UNLOCK(); | ||||
} | } | ||||
/* | /* | ||||
* Fill filter statistics | * Fill filter statistics | ||||
*/ | */ | ||||
static void | static void | ||||
bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd) | bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd) | ||||
{ | { | ||||
BPF_LOCK_ASSERT(); | |||||
bzero(d, sizeof(*d)); | bzero(d, sizeof(*d)); | ||||
BPFD_LOCK_ASSERT(bd); | |||||
d->bd_structsize = sizeof(*d); | d->bd_structsize = sizeof(*d); | ||||
/* XXX: reading should be protected by global lock */ | |||||
d->bd_immediate = bd->bd_immediate; | d->bd_immediate = bd->bd_immediate; | ||||
d->bd_promisc = bd->bd_promisc; | d->bd_promisc = bd->bd_promisc; | ||||
d->bd_hdrcmplt = bd->bd_hdrcmplt; | d->bd_hdrcmplt = bd->bd_hdrcmplt; | ||||
d->bd_direction = bd->bd_direction; | d->bd_direction = bd->bd_direction; | ||||
d->bd_feedback = bd->bd_feedback; | d->bd_feedback = bd->bd_feedback; | ||||
d->bd_async = bd->bd_async; | d->bd_async = bd->bd_async; | ||||
d->bd_rcount = counter_u64_fetch(bd->bd_rcount); | d->bd_rcount = counter_u64_fetch(bd->bd_rcount); | ||||
d->bd_dcount = counter_u64_fetch(bd->bd_dcount); | d->bd_dcount = counter_u64_fetch(bd->bd_dcount); | ||||
▲ Show 20 Lines • Show All 58 Lines • ▼ Show 20 Lines | bpf_stats_sysctl(SYSCTL_HANDLER_ARGS) | ||||
xbdbuf = malloc(req->oldlen, M_BPF, M_WAITOK); | xbdbuf = malloc(req->oldlen, M_BPF, M_WAITOK); | ||||
BPF_LOCK(); | BPF_LOCK(); | ||||
if (req->oldlen < (bpf_bpfd_cnt * sizeof(*xbd))) { | if (req->oldlen < (bpf_bpfd_cnt * sizeof(*xbd))) { | ||||
BPF_UNLOCK(); | BPF_UNLOCK(); | ||||
free(xbdbuf, M_BPF); | free(xbdbuf, M_BPF); | ||||
return (ENOMEM); | return (ENOMEM); | ||||
} | } | ||||
index = 0; | index = 0; | ||||
LIST_FOREACH(bp, &bpf_iflist, bif_next) { | CK_LIST_FOREACH(bp, &bpf_iflist, bif_next) { | ||||
BPFIF_RLOCK(bp); | |||||
/* Send writers-only first */ | /* Send writers-only first */ | ||||
LIST_FOREACH(bd, &bp->bif_wlist, bd_next) { | CK_LIST_FOREACH(bd, &bp->bif_wlist, bd_next) { | ||||
xbd = &xbdbuf[index++]; | xbd = &xbdbuf[index++]; | ||||
BPFD_LOCK(bd); | |||||
bpfstats_fill_xbpf(xbd, bd); | bpfstats_fill_xbpf(xbd, bd); | ||||
BPFD_UNLOCK(bd); | |||||
} | } | ||||
LIST_FOREACH(bd, &bp->bif_dlist, bd_next) { | CK_LIST_FOREACH(bd, &bp->bif_dlist, bd_next) { | ||||
xbd = &xbdbuf[index++]; | xbd = &xbdbuf[index++]; | ||||
BPFD_LOCK(bd); | |||||
bpfstats_fill_xbpf(xbd, bd); | bpfstats_fill_xbpf(xbd, bd); | ||||
BPFD_UNLOCK(bd); | |||||
} | } | ||||
BPFIF_RUNLOCK(bp); | |||||
} | } | ||||
BPF_UNLOCK(); | BPF_UNLOCK(); | ||||
error = SYSCTL_OUT(req, xbdbuf, index * sizeof(*xbd)); | error = SYSCTL_OUT(req, xbdbuf, index * sizeof(*xbd)); | ||||
free(xbdbuf, M_BPF); | free(xbdbuf, M_BPF); | ||||
return (error); | return (error); | ||||
} | } | ||||
SYSINIT(bpfdev,SI_SUB_DRIVERS,SI_ORDER_MIDDLE,bpf_drvinit,NULL); | SYSINIT(bpfdev,SI_SUB_DRIVERS,SI_ORDER_MIDDLE,bpf_drvinit,NULL); | ||||
▲ Show 20 Lines • Show All 63 Lines • ▼ Show 20 Lines | bpf_show_bpf_if(struct bpf_if *bpf_if) | ||||
if (bpf_if == NULL) | if (bpf_if == NULL) | ||||
return; | return; | ||||
db_printf("%p:\n", bpf_if); | db_printf("%p:\n", bpf_if); | ||||
#define BPF_DB_PRINTF(f, e) db_printf(" %s = " f "\n", #e, bpf_if->e); | #define BPF_DB_PRINTF(f, e) db_printf(" %s = " f "\n", #e, bpf_if->e); | ||||
/* bif_ext.bif_next */ | /* bif_ext.bif_next */ | ||||
/* bif_ext.bif_dlist */ | /* bif_ext.bif_dlist */ | ||||
BPF_DB_PRINTF("%#x", bif_dlt); | BPF_DB_PRINTF("%#x", bif_dlt); | ||||
BPF_DB_PRINTF("%u", bif_hdrlen); | BPF_DB_PRINTF("%u", bif_hdrlen); | ||||
BPF_DB_PRINTF("%p", bif_ifp); | |||||
/* bif_lock */ | |||||
/* bif_wlist */ | /* bif_wlist */ | ||||
BPF_DB_PRINTF("%#x", bif_flags); | BPF_DB_PRINTF("%p", bif_ifp); | ||||
BPF_DB_PRINTF("%p", bif_bpf); | |||||
BPF_DB_PRINTF("%u", bif_refcnt); | |||||
} | } | ||||
DB_SHOW_COMMAND(bpf_if, db_show_bpf_if) | DB_SHOW_COMMAND(bpf_if, db_show_bpf_if) | ||||
{ | { | ||||
if (!have_addr) { | if (!have_addr) { | ||||
db_printf("usage: show bpf_if <struct bpf_if *>\n"); | db_printf("usage: show bpf_if <struct bpf_if *>\n"); | ||||
return; | return; | ||||
} | } | ||||
bpf_show_bpf_if((struct bpf_if *)addr); | bpf_show_bpf_if((struct bpf_if *)addr); | ||||
} | } | ||||
#endif | #endif |