diff --git a/sys/kern/kern_mbuf.c b/sys/kern/kern_mbuf.c
index 23050e991418..f1e76ef00c65 100644
--- a/sys/kern/kern_mbuf.c
+++ b/sys/kern/kern_mbuf.c
@@ -1,1764 +1,1737 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2004, 2005,
* Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice unmodified, this list of conditions and the following
* disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_param.h"
#include "opt_kern_tls.h"
#include <sys/param.h>
#include <sys/conf.h>
#include <sys/domainset.h>
#include <sys/malloc.h>
#include <sys/systm.h>
#include <sys/mbuf.h>
#include <sys/domain.h>
#include <sys/eventhandler.h>
#include <sys/kernel.h>
#include <sys/ktls.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/protosw.h>
#include <sys/refcount.h>
#include <sys/sf_buf.h>
#include <sys/smp.h>
#include <sys/socket.h>
#include <sys/sysctl.h>
#include <net/if.h>
#include <net/if_var.h>
#include <vm/vm.h>
#include <vm/vm_extern.h>
#include <vm/vm_kern.h>
#include <vm/vm_page.h>
#include <vm/vm_pageout.h>
#include <vm/vm_map.h>
#include <vm/uma.h>
#include <vm/uma_dbg.h>
/*
* In FreeBSD, Mbufs and Mbuf Clusters are allocated from UMA
* Zones.
*
* Mbuf Clusters (2K, contiguous) are allocated from the Cluster
* Zone. The Zone can be capped at kern.ipc.nmbclusters, if the
* administrator so desires.
*
* Mbufs are allocated from a UMA Primary Zone called the Mbuf
* Zone.
*
* Additionally, FreeBSD provides a Packet Zone, which it
* configures as a Secondary Zone to the Mbuf Primary Zone,
* thus sharing backend Slab kegs with the Mbuf Primary Zone.
*
* Thus common-case allocations and locking are simplified:
*
* m_clget() m_getcl()
* | |
* | .------------>[(Packet Cache)] m_get(), m_gethdr()
* | | [ Packet ] |
* [(Cluster Cache)] [ Secondary ] [ (Mbuf Cache) ]
* [ Cluster Zone ] [ Zone ] [ Mbuf Primary Zone ]
* | \________ |
* [ Cluster Keg ] \ /
* | [ Mbuf Keg ]
* [ Cluster Slabs ] |
* | [ Mbuf Slabs ]
* \____________(VM)_________________/
*
*
* Whenever an object is allocated with uma_zalloc() out of
* one of the Zones its _ctor_ function is executed. The same
* for any deallocation through uma_zfree() the _dtor_ function
* is executed.
*
* Caches are per-CPU and are filled from the Primary Zone.
*
* Whenever an object is allocated from the underlying global
* memory pool it gets pre-initialized with the _zinit_ functions.
* When the Keg's are overfull objects get decommissioned with
* _zfini_ functions and free'd back to the global memory pool.
*
*/
int nmbufs; /* limits number of mbufs */
int nmbclusters; /* limits number of mbuf clusters */
int nmbjumbop; /* limits number of page size jumbo clusters */
int nmbjumbo9; /* limits number of 9k jumbo clusters */
int nmbjumbo16; /* limits number of 16k jumbo clusters */
bool mb_use_ext_pgs = false; /* use M_EXTPG mbufs for sendfile & TLS */
static int
sysctl_mb_use_ext_pgs(SYSCTL_HANDLER_ARGS)
{
int error, extpg;
extpg = mb_use_ext_pgs;
error = sysctl_handle_int(oidp, &extpg, 0, req);
if (error == 0 && req->newptr != NULL) {
if (extpg != 0 && !PMAP_HAS_DMAP)
error = EOPNOTSUPP;
else
mb_use_ext_pgs = extpg != 0;
}
return (error);
}
SYSCTL_PROC(_kern_ipc, OID_AUTO, mb_use_ext_pgs, CTLTYPE_INT | CTLFLAG_RW,
&mb_use_ext_pgs, 0,
sysctl_mb_use_ext_pgs, "IU",
"Use unmapped mbufs for sendfile(2) and TLS offload");
static quad_t maxmbufmem; /* overall real memory limit for all mbufs */
SYSCTL_QUAD(_kern_ipc, OID_AUTO, maxmbufmem, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &maxmbufmem, 0,
"Maximum real memory allocatable to various mbuf types");
static counter_u64_t snd_tag_count;
SYSCTL_COUNTER_U64(_kern_ipc, OID_AUTO, num_snd_tags, CTLFLAG_RW,
&snd_tag_count, "# of active mbuf send tags");
/*
* tunable_mbinit() has to be run before any mbuf allocations are done.
*/
static void
tunable_mbinit(void *dummy)
{
quad_t realmem;
int extpg;
/*
* The default limit for all mbuf related memory is 1/2 of all
* available kernel memory (physical or kmem).
* At most it can be 3/4 of available kernel memory.
*/
realmem = qmin((quad_t)physmem * PAGE_SIZE, vm_kmem_size);
maxmbufmem = realmem / 2;
TUNABLE_QUAD_FETCH("kern.ipc.maxmbufmem", &maxmbufmem);
if (maxmbufmem > realmem / 4 * 3)
maxmbufmem = realmem / 4 * 3;
TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters);
if (nmbclusters == 0)
nmbclusters = maxmbufmem / MCLBYTES / 4;
TUNABLE_INT_FETCH("kern.ipc.nmbjumbop", &nmbjumbop);
if (nmbjumbop == 0)
nmbjumbop = maxmbufmem / MJUMPAGESIZE / 4;
TUNABLE_INT_FETCH("kern.ipc.nmbjumbo9", &nmbjumbo9);
if (nmbjumbo9 == 0)
nmbjumbo9 = maxmbufmem / MJUM9BYTES / 6;
TUNABLE_INT_FETCH("kern.ipc.nmbjumbo16", &nmbjumbo16);
if (nmbjumbo16 == 0)
nmbjumbo16 = maxmbufmem / MJUM16BYTES / 6;
/*
* We need at least as many mbufs as we have clusters of
* the various types added together.
*/
TUNABLE_INT_FETCH("kern.ipc.nmbufs", &nmbufs);
if (nmbufs < nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16)
nmbufs = lmax(maxmbufmem / MSIZE / 5,
nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16);
/*
* Unmapped mbufs can only safely be used on platforms with a direct
* map.
*/
if (PMAP_HAS_DMAP) {
extpg = 1;
TUNABLE_INT_FETCH("kern.ipc.mb_use_ext_pgs", &extpg);
mb_use_ext_pgs = extpg != 0;
}
}
SYSINIT(tunable_mbinit, SI_SUB_KMEM, SI_ORDER_MIDDLE, tunable_mbinit, NULL);
static int
sysctl_nmbclusters(SYSCTL_HANDLER_ARGS)
{
int error, newnmbclusters;
newnmbclusters = nmbclusters;
error = sysctl_handle_int(oidp, &newnmbclusters, 0, req);
if (error == 0 && req->newptr && newnmbclusters != nmbclusters) {
if (newnmbclusters > nmbclusters &&
nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
nmbclusters = newnmbclusters;
nmbclusters = uma_zone_set_max(zone_clust, nmbclusters);
EVENTHANDLER_INVOKE(nmbclusters_change);
} else
error = EINVAL;
}
return (error);
}
SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbclusters,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, &nmbclusters, 0,
sysctl_nmbclusters, "IU",
"Maximum number of mbuf clusters allowed");
static int
sysctl_nmbjumbop(SYSCTL_HANDLER_ARGS)
{
int error, newnmbjumbop;
newnmbjumbop = nmbjumbop;
error = sysctl_handle_int(oidp, &newnmbjumbop, 0, req);
if (error == 0 && req->newptr && newnmbjumbop != nmbjumbop) {
if (newnmbjumbop > nmbjumbop &&
nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
nmbjumbop = newnmbjumbop;
nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop);
} else
error = EINVAL;
}
return (error);
}
SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbop,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, &nmbjumbop, 0,
sysctl_nmbjumbop, "IU",
"Maximum number of mbuf page size jumbo clusters allowed");
static int
sysctl_nmbjumbo9(SYSCTL_HANDLER_ARGS)
{
int error, newnmbjumbo9;
newnmbjumbo9 = nmbjumbo9;
error = sysctl_handle_int(oidp, &newnmbjumbo9, 0, req);
if (error == 0 && req->newptr && newnmbjumbo9 != nmbjumbo9) {
if (newnmbjumbo9 > nmbjumbo9 &&
nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
nmbjumbo9 = newnmbjumbo9;
nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9);
} else
error = EINVAL;
}
return (error);
}
SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo9,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, &nmbjumbo9, 0,
sysctl_nmbjumbo9, "IU",
"Maximum number of mbuf 9k jumbo clusters allowed");
static int
sysctl_nmbjumbo16(SYSCTL_HANDLER_ARGS)
{
int error, newnmbjumbo16;
newnmbjumbo16 = nmbjumbo16;
error = sysctl_handle_int(oidp, &newnmbjumbo16, 0, req);
if (error == 0 && req->newptr && newnmbjumbo16 != nmbjumbo16) {
if (newnmbjumbo16 > nmbjumbo16 &&
nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
nmbjumbo16 = newnmbjumbo16;
nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16);
} else
error = EINVAL;
}
return (error);
}
SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo16,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, &nmbjumbo16, 0,
sysctl_nmbjumbo16, "IU",
"Maximum number of mbuf 16k jumbo clusters allowed");
static int
sysctl_nmbufs(SYSCTL_HANDLER_ARGS)
{
int error, newnmbufs;
newnmbufs = nmbufs;
error = sysctl_handle_int(oidp, &newnmbufs, 0, req);
if (error == 0 && req->newptr && newnmbufs != nmbufs) {
if (newnmbufs > nmbufs) {
nmbufs = newnmbufs;
nmbufs = uma_zone_set_max(zone_mbuf, nmbufs);
EVENTHANDLER_INVOKE(nmbufs_change);
} else
error = EINVAL;
}
return (error);
}
SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbufs,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
&nmbufs, 0, sysctl_nmbufs, "IU",
"Maximum number of mbufs allowed");
/*
* Zones from which we allocate.
*/
uma_zone_t zone_mbuf;
uma_zone_t zone_clust;
uma_zone_t zone_pack;
uma_zone_t zone_jumbop;
uma_zone_t zone_jumbo9;
uma_zone_t zone_jumbo16;
/*
* Local prototypes.
*/
static int mb_ctor_mbuf(void *, int, void *, int);
static int mb_ctor_clust(void *, int, void *, int);
static int mb_ctor_pack(void *, int, void *, int);
static void mb_dtor_mbuf(void *, int, void *);
static void mb_dtor_pack(void *, int, void *);
static int mb_zinit_pack(void *, int, int);
static void mb_zfini_pack(void *, int);
static void mb_reclaim(uma_zone_t, int);
/* Ensure that MSIZE is a power of 2. */
CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
_Static_assert(sizeof(struct mbuf) <= MSIZE,
"size of mbuf exceeds MSIZE");
/*
* Initialize FreeBSD Network buffer allocation.
*/
static void
mbuf_init(void *dummy)
{
/*
* Configure UMA zones for Mbufs, Clusters, and Packets.
*/
zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE,
mb_ctor_mbuf, mb_dtor_mbuf, NULL, NULL,
MSIZE - 1, UMA_ZONE_CONTIG | UMA_ZONE_MAXBUCKET);
if (nmbufs > 0)
nmbufs = uma_zone_set_max(zone_mbuf, nmbufs);
uma_zone_set_warning(zone_mbuf, "kern.ipc.nmbufs limit reached");
uma_zone_set_maxaction(zone_mbuf, mb_reclaim);
zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES,
mb_ctor_clust, NULL, NULL, NULL,
UMA_ALIGN_PTR, UMA_ZONE_CONTIG);
if (nmbclusters > 0)
nmbclusters = uma_zone_set_max(zone_clust, nmbclusters);
uma_zone_set_warning(zone_clust, "kern.ipc.nmbclusters limit reached");
uma_zone_set_maxaction(zone_clust, mb_reclaim);
zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,
mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf);
/* Make jumbo frame zone too. Page size, 9k and 16k. */
zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE,
mb_ctor_clust, NULL, NULL, NULL,
UMA_ALIGN_PTR, UMA_ZONE_CONTIG);
if (nmbjumbop > 0)
nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop);
uma_zone_set_warning(zone_jumbop, "kern.ipc.nmbjumbop limit reached");
uma_zone_set_maxaction(zone_jumbop, mb_reclaim);
zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES,
mb_ctor_clust, NULL, NULL, NULL,
UMA_ALIGN_PTR, UMA_ZONE_CONTIG);
if (nmbjumbo9 > 0)
nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9);
uma_zone_set_warning(zone_jumbo9, "kern.ipc.nmbjumbo9 limit reached");
uma_zone_set_maxaction(zone_jumbo9, mb_reclaim);
zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES,
mb_ctor_clust, NULL, NULL, NULL,
UMA_ALIGN_PTR, UMA_ZONE_CONTIG);
if (nmbjumbo16 > 0)
nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16);
uma_zone_set_warning(zone_jumbo16, "kern.ipc.nmbjumbo16 limit reached");
uma_zone_set_maxaction(zone_jumbo16, mb_reclaim);
/*
* Hook event handler for low-memory situation, used to
* drain protocols and push data back to the caches (UMA
* later pushes it back to VM).
*/
EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL,
EVENTHANDLER_PRI_FIRST);
snd_tag_count = counter_u64_alloc(M_WAITOK);
}
SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL);
#ifdef DEBUGNET
/*
* debugnet makes use of a pre-allocated pool of mbufs and clusters. When
* debugnet is configured, we initialize a set of UMA cache zones which return
* items from this pool. At panic-time, the regular UMA zone pointers are
* overwritten with those of the cache zones so that drivers may allocate and
* free mbufs and clusters without attempting to allocate physical memory.
*
* We keep mbufs and clusters in a pair of mbuf queues. In particular, for
* the purpose of caching clusters, we treat them as mbufs.
*/
static struct mbufq dn_mbufq =
{ STAILQ_HEAD_INITIALIZER(dn_mbufq.mq_head), 0, INT_MAX };
static struct mbufq dn_clustq =
{ STAILQ_HEAD_INITIALIZER(dn_clustq.mq_head), 0, INT_MAX };
static int dn_clsize;
static uma_zone_t dn_zone_mbuf;
static uma_zone_t dn_zone_clust;
static uma_zone_t dn_zone_pack;
static struct debugnet_saved_zones {
uma_zone_t dsz_mbuf;
uma_zone_t dsz_clust;
uma_zone_t dsz_pack;
uma_zone_t dsz_jumbop;
uma_zone_t dsz_jumbo9;
uma_zone_t dsz_jumbo16;
bool dsz_debugnet_zones_enabled;
} dn_saved_zones;
static int
dn_buf_import(void *arg, void **store, int count, int domain __unused,
int flags)
{
struct mbufq *q;
struct mbuf *m;
int i;
q = arg;
for (i = 0; i < count; i++) {
m = mbufq_dequeue(q);
if (m == NULL)
break;
trash_init(m, q == &dn_mbufq ? MSIZE : dn_clsize, flags);
store[i] = m;
}
KASSERT((flags & M_WAITOK) == 0 || i == count,
("%s: ran out of pre-allocated mbufs", __func__));
return (i);
}
static void
dn_buf_release(void *arg, void **store, int count)
{
struct mbufq *q;
struct mbuf *m;
int i;
q = arg;
for (i = 0; i < count; i++) {
m = store[i];
(void)mbufq_enqueue(q, m);
}
}
static int
dn_pack_import(void *arg __unused, void **store, int count, int domain __unused,
int flags __unused)
{
struct mbuf *m;
void *clust;
int i;
for (i = 0; i < count; i++) {
m = m_get(MT_DATA, M_NOWAIT);
if (m == NULL)
break;
clust = uma_zalloc(dn_zone_clust, M_NOWAIT);
if (clust == NULL) {
m_free(m);
break;
}
mb_ctor_clust(clust, dn_clsize, m, 0);
store[i] = m;
}
KASSERT((flags & M_WAITOK) == 0 || i == count,
("%s: ran out of pre-allocated mbufs", __func__));
return (i);
}
static void
dn_pack_release(void *arg __unused, void **store, int count)
{
struct mbuf *m;
void *clust;
int i;
for (i = 0; i < count; i++) {
m = store[i];
clust = m->m_ext.ext_buf;
uma_zfree(dn_zone_clust, clust);
uma_zfree(dn_zone_mbuf, m);
}
}
/*
* Free the pre-allocated mbufs and clusters reserved for debugnet, and destroy
* the corresponding UMA cache zones.
*/
void
debugnet_mbuf_drain(void)
{
struct mbuf *m;
void *item;
if (dn_zone_mbuf != NULL) {
uma_zdestroy(dn_zone_mbuf);
dn_zone_mbuf = NULL;
}
if (dn_zone_clust != NULL) {
uma_zdestroy(dn_zone_clust);
dn_zone_clust = NULL;
}
if (dn_zone_pack != NULL) {
uma_zdestroy(dn_zone_pack);
dn_zone_pack = NULL;
}
while ((m = mbufq_dequeue(&dn_mbufq)) != NULL)
m_free(m);
while ((item = mbufq_dequeue(&dn_clustq)) != NULL)
uma_zfree(m_getzone(dn_clsize), item);
}
/*
* Callback invoked immediately prior to starting a debugnet connection.
*/
void
debugnet_mbuf_start(void)
{
MPASS(!dn_saved_zones.dsz_debugnet_zones_enabled);
/* Save the old zone pointers to restore when debugnet is closed. */
dn_saved_zones = (struct debugnet_saved_zones) {
.dsz_debugnet_zones_enabled = true,
.dsz_mbuf = zone_mbuf,
.dsz_clust = zone_clust,
.dsz_pack = zone_pack,
.dsz_jumbop = zone_jumbop,
.dsz_jumbo9 = zone_jumbo9,
.dsz_jumbo16 = zone_jumbo16,
};
/*
* All cluster zones return buffers of the size requested by the
* drivers. It's up to the driver to reinitialize the zones if the
* MTU of a debugnet-enabled interface changes.
*/
printf("debugnet: overwriting mbuf zone pointers\n");
zone_mbuf = dn_zone_mbuf;
zone_clust = dn_zone_clust;
zone_pack = dn_zone_pack;
zone_jumbop = dn_zone_clust;
zone_jumbo9 = dn_zone_clust;
zone_jumbo16 = dn_zone_clust;
}
/*
* Callback invoked when a debugnet connection is closed/finished.
*/
void
debugnet_mbuf_finish(void)
{
MPASS(dn_saved_zones.dsz_debugnet_zones_enabled);
printf("debugnet: restoring mbuf zone pointers\n");
zone_mbuf = dn_saved_zones.dsz_mbuf;
zone_clust = dn_saved_zones.dsz_clust;
zone_pack = dn_saved_zones.dsz_pack;
zone_jumbop = dn_saved_zones.dsz_jumbop;
zone_jumbo9 = dn_saved_zones.dsz_jumbo9;
zone_jumbo16 = dn_saved_zones.dsz_jumbo16;
memset(&dn_saved_zones, 0, sizeof(dn_saved_zones));
}
/*
* Reinitialize the debugnet mbuf+cluster pool and cache zones.
*/
void
debugnet_mbuf_reinit(int nmbuf, int nclust, int clsize)
{
struct mbuf *m;
void *item;
debugnet_mbuf_drain();
dn_clsize = clsize;
dn_zone_mbuf = uma_zcache_create("debugnet_" MBUF_MEM_NAME,
MSIZE, mb_ctor_mbuf, mb_dtor_mbuf, NULL, NULL,
dn_buf_import, dn_buf_release,
&dn_mbufq, UMA_ZONE_NOBUCKET);
dn_zone_clust = uma_zcache_create("debugnet_" MBUF_CLUSTER_MEM_NAME,
clsize, mb_ctor_clust, NULL, NULL, NULL,
dn_buf_import, dn_buf_release,
&dn_clustq, UMA_ZONE_NOBUCKET);
dn_zone_pack = uma_zcache_create("debugnet_" MBUF_PACKET_MEM_NAME,
MCLBYTES, mb_ctor_pack, mb_dtor_pack, NULL, NULL,
dn_pack_import, dn_pack_release,
NULL, UMA_ZONE_NOBUCKET);
while (nmbuf-- > 0) {
m = m_get(MT_DATA, M_WAITOK);
uma_zfree(dn_zone_mbuf, m);
}
while (nclust-- > 0) {
item = uma_zalloc(m_getzone(dn_clsize), M_WAITOK);
uma_zfree(dn_zone_clust, item);
}
}
#endif /* DEBUGNET */
/*
* Constructor for Mbuf primary zone.
*
* The 'arg' pointer points to a mb_args structure which
* contains call-specific information required to support the
* mbuf allocation API. See mbuf.h.
*/
static int
mb_ctor_mbuf(void *mem, int size, void *arg, int how)
{
struct mbuf *m;
struct mb_args *args;
int error;
int flags;
short type;
args = (struct mb_args *)arg;
type = args->type;
/*
* The mbuf is initialized later. The caller has the
* responsibility to set up any MAC labels too.
*/
if (type == MT_NOINIT)
return (0);
m = (struct mbuf *)mem;
flags = args->flags;
MPASS((flags & M_NOFREE) == 0);
error = m_init(m, how, type, flags);
return (error);
}
/*
* The Mbuf primary zone destructor.
*/
static void
mb_dtor_mbuf(void *mem, int size, void *arg)
{
struct mbuf *m;
unsigned long flags __diagused;
m = (struct mbuf *)mem;
flags = (unsigned long)arg;
KASSERT((m->m_flags & M_NOFREE) == 0, ("%s: M_NOFREE set", __func__));
KASSERT((flags & 0x1) == 0, ("%s: obsolete MB_DTOR_SKIP passed", __func__));
if ((m->m_flags & M_PKTHDR) && !SLIST_EMPTY(&m->m_pkthdr.tags))
m_tag_delete_chain(m, NULL);
}
/*
* The Mbuf Packet zone destructor.
*/
static void
mb_dtor_pack(void *mem, int size, void *arg)
{
struct mbuf *m;
m = (struct mbuf *)mem;
if ((m->m_flags & M_PKTHDR) != 0)
m_tag_delete_chain(m, NULL);
/* Make sure we've got a clean cluster back. */
KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__));
KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__));
KASSERT(m->m_ext.ext_arg1 == NULL, ("%s: ext_arg1 != NULL", __func__));
KASSERT(m->m_ext.ext_arg2 == NULL, ("%s: ext_arg2 != NULL", __func__));
KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__));
KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__));
#if defined(INVARIANTS) && !defined(KMSAN)
trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg);
#endif
/*
* If there are processes blocked on zone_clust, waiting for pages
* to be freed up, cause them to be woken up by draining the
* packet zone. We are exposed to a race here (in the check for
* the UMA_ZFLAG_FULL) where we might miss the flag set, but that
* is deliberate. We don't want to acquire the zone lock for every
* mbuf free.
*/
if (uma_zone_exhausted(zone_clust))
uma_zone_reclaim(zone_pack, UMA_RECLAIM_DRAIN);
}
/*
* The Cluster and Jumbo[PAGESIZE|9|16] zone constructor.
*
* Here the 'arg' pointer points to the Mbuf which we
* are configuring cluster storage for. If 'arg' is
* empty we allocate just the cluster without setting
* the mbuf to it. See mbuf.h.
*/
static int
mb_ctor_clust(void *mem, int size, void *arg, int how)
{
struct mbuf *m;
m = (struct mbuf *)arg;
if (m != NULL) {
m->m_ext.ext_buf = (char *)mem;
m->m_data = m->m_ext.ext_buf;
m->m_flags |= M_EXT;
m->m_ext.ext_free = NULL;
m->m_ext.ext_arg1 = NULL;
m->m_ext.ext_arg2 = NULL;
m->m_ext.ext_size = size;
m->m_ext.ext_type = m_gettype(size);
m->m_ext.ext_flags = EXT_FLAG_EMBREF;
m->m_ext.ext_count = 1;
}
return (0);
}
/*
* The Packet secondary zone's init routine, executed on the
* object's transition from mbuf keg slab to zone cache.
*/
static int
mb_zinit_pack(void *mem, int size, int how)
{
struct mbuf *m;
m = (struct mbuf *)mem; /* m is virgin. */
if (uma_zalloc_arg(zone_clust, m, how) == NULL ||
m->m_ext.ext_buf == NULL)
return (ENOMEM);
m->m_ext.ext_type = EXT_PACKET; /* Override. */
#if defined(INVARIANTS) && !defined(KMSAN)
trash_init(m->m_ext.ext_buf, MCLBYTES, how);
#endif
return (0);
}
/*
* The Packet secondary zone's fini routine, executed on the
* object's transition from zone cache to keg slab.
*/
static void
mb_zfini_pack(void *mem, int size)
{
struct mbuf *m;
m = (struct mbuf *)mem;
#if defined(INVARIANTS) && !defined(KMSAN)
trash_fini(m->m_ext.ext_buf, MCLBYTES);
#endif
uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL);
#if defined(INVARIANTS) && !defined(KMSAN)
trash_dtor(mem, size, NULL);
#endif
}
/*
* The "packet" keg constructor.
*/
static int
mb_ctor_pack(void *mem, int size, void *arg, int how)
{
struct mbuf *m;
struct mb_args *args;
int error, flags;
short type;
m = (struct mbuf *)mem;
args = (struct mb_args *)arg;
flags = args->flags;
type = args->type;
MPASS((flags & M_NOFREE) == 0);
#if defined(INVARIANTS) && !defined(KMSAN)
trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how);
#endif
error = m_init(m, how, type, flags);
/* m_ext is already initialized. */
m->m_data = m->m_ext.ext_buf;
m->m_flags = (flags | M_EXT);
return (error);
}
/*
* This is the protocol drain routine. Called by UMA whenever any of the
* mbuf zones is closed to its limit.
*
* No locks should be held when this is called. The drain routines have to
* presently acquire some locks which raises the possibility of lock order
* reversal.
*/
static void
mb_reclaim(uma_zone_t zone __unused, int pending __unused)
{
struct epoch_tracker et;
struct domain *dp;
struct protosw *pr;
WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL, __func__);
NET_EPOCH_ENTER(et);
for (dp = domains; dp != NULL; dp = dp->dom_next)
for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
if (pr->pr_drain != NULL)
(*pr->pr_drain)();
NET_EPOCH_EXIT(et);
}
/*
* Free "count" units of I/O from an mbuf chain. They could be held
* in M_EXTPG or just as a normal mbuf. This code is intended to be
* called in an error path (I/O error, closed connection, etc).
*/
void
mb_free_notready(struct mbuf *m, int count)
{
int i;
for (i = 0; i < count && m != NULL; i++) {
if ((m->m_flags & M_EXTPG) != 0) {
m->m_epg_nrdy--;
if (m->m_epg_nrdy != 0)
continue;
}
m = m_free(m);
}
KASSERT(i == count, ("Removed only %d items from %p", i, m));
}
/*
* Compress an unmapped mbuf into a simple mbuf when it holds a small
* amount of data. This is used as a DOS defense to avoid having
* small packets tie up wired pages, an ext_pgs structure, and an
* mbuf. Since this converts the existing mbuf in place, it can only
* be used if there are no other references to 'm'.
*/
int
mb_unmapped_compress(struct mbuf *m)
{
volatile u_int *refcnt;
char buf[MLEN];
/*
* Assert that 'm' does not have a packet header. If 'm' had
* a packet header, it would only be able to hold MHLEN bytes
* and m_data would have to be initialized differently.
*/
KASSERT((m->m_flags & M_PKTHDR) == 0 && (m->m_flags & M_EXTPG),
("%s: m %p !M_EXTPG or M_PKTHDR", __func__, m));
KASSERT(m->m_len <= MLEN, ("m_len too large %p", m));
if (m->m_ext.ext_flags & EXT_FLAG_EMBREF) {
refcnt = &m->m_ext.ext_count;
} else {
KASSERT(m->m_ext.ext_cnt != NULL,
("%s: no refcounting pointer on %p", __func__, m));
refcnt = m->m_ext.ext_cnt;
}
if (*refcnt != 1)
return (EBUSY);
m_copydata(m, 0, m->m_len, buf);
/* Free the backing pages. */
m->m_ext.ext_free(m);
/* Turn 'm' into a "normal" mbuf. */
m->m_flags &= ~(M_EXT | M_RDONLY | M_EXTPG);
m->m_data = m->m_dat;
/* Copy data back into m. */
bcopy(buf, mtod(m, char *), m->m_len);
return (0);
}
/*
* These next few routines are used to permit downgrading an unmapped
* mbuf to a chain of mapped mbufs. This is used when an interface
* doesn't supported unmapped mbufs or if checksums need to be
* computed in software.
*
* Each unmapped mbuf is converted to a chain of mbufs. First, any
* TLS header data is stored in a regular mbuf. Second, each page of
* unmapped data is stored in an mbuf with an EXT_SFBUF external
* cluster. These mbufs use an sf_buf to provide a valid KVA for the
* associated physical page. They also hold a reference on the
* original M_EXTPG mbuf to ensure the physical page doesn't go away.
* Finally, any TLS trailer data is stored in a regular mbuf.
*
* mb_unmapped_free_mext() is the ext_free handler for the EXT_SFBUF
* mbufs. It frees the associated sf_buf and releases its reference
* on the original M_EXTPG mbuf.
*
* _mb_unmapped_to_ext() is a helper function that converts a single
* unmapped mbuf into a chain of mbufs.
*
* mb_unmapped_to_ext() is the public function that walks an mbuf
* chain converting any unmapped mbufs to mapped mbufs. It returns
* the new chain of unmapped mbufs on success. On failure it frees
* the original mbuf chain and returns NULL.
*/
static void
mb_unmapped_free_mext(struct mbuf *m)
{
struct sf_buf *sf;
struct mbuf *old_m;
sf = m->m_ext.ext_arg1;
sf_buf_free(sf);
/* Drop the reference on the backing M_EXTPG mbuf. */
old_m = m->m_ext.ext_arg2;
mb_free_extpg(old_m);
}
static struct mbuf *
_mb_unmapped_to_ext(struct mbuf *m)
{
struct mbuf *m_new, *top, *prev, *mref;
struct sf_buf *sf;
vm_page_t pg;
int i, len, off, pglen, pgoff, seglen, segoff;
volatile u_int *refcnt;
u_int ref_inc = 0;
M_ASSERTEXTPG(m);
len = m->m_len;
KASSERT(m->m_epg_tls == NULL, ("%s: can't convert TLS mbuf %p",
__func__, m));
/* See if this is the mbuf that holds the embedded refcount. */
if (m->m_ext.ext_flags & EXT_FLAG_EMBREF) {
refcnt = &m->m_ext.ext_count;
mref = m;
} else {
KASSERT(m->m_ext.ext_cnt != NULL,
("%s: no refcounting pointer on %p", __func__, m));
refcnt = m->m_ext.ext_cnt;
mref = __containerof(refcnt, struct mbuf, m_ext.ext_count);
}
/* Skip over any data removed from the front. */
off = mtod(m, vm_offset_t);
top = NULL;
if (m->m_epg_hdrlen != 0) {
if (off >= m->m_epg_hdrlen) {
off -= m->m_epg_hdrlen;
} else {
seglen = m->m_epg_hdrlen - off;
segoff = off;
seglen = min(seglen, len);
off = 0;
len -= seglen;
m_new = m_get(M_NOWAIT, MT_DATA);
if (m_new == NULL)
goto fail;
m_new->m_len = seglen;
prev = top = m_new;
memcpy(mtod(m_new, void *), &m->m_epg_hdr[segoff],
seglen);
}
}
pgoff = m->m_epg_1st_off;
for (i = 0; i < m->m_epg_npgs && len > 0; i++) {
pglen = m_epg_pagelen(m, i, pgoff);
if (off >= pglen) {
off -= pglen;
pgoff = 0;
continue;
}
seglen = pglen - off;
segoff = pgoff + off;
off = 0;
seglen = min(seglen, len);
len -= seglen;
pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
m_new = m_get(M_NOWAIT, MT_DATA);
if (m_new == NULL)
goto fail;
if (top == NULL) {
top = prev = m_new;
} else {
prev->m_next = m_new;
prev = m_new;
}
sf = sf_buf_alloc(pg, SFB_NOWAIT);
if (sf == NULL)
goto fail;
ref_inc++;
m_extadd(m_new, (char *)sf_buf_kva(sf), PAGE_SIZE,
mb_unmapped_free_mext, sf, mref, M_RDONLY, EXT_SFBUF);
m_new->m_data += segoff;
m_new->m_len = seglen;
pgoff = 0;
};
if (len != 0) {
KASSERT((off + len) <= m->m_epg_trllen,
("off + len > trail (%d + %d > %d)", off, len,
m->m_epg_trllen));
m_new = m_get(M_NOWAIT, MT_DATA);
if (m_new == NULL)
goto fail;
if (top == NULL)
top = m_new;
else
prev->m_next = m_new;
m_new->m_len = len;
memcpy(mtod(m_new, void *), &m->m_epg_trail[off], len);
}
if (ref_inc != 0) {
/*
* Obtain an additional reference on the old mbuf for
* each created EXT_SFBUF mbuf. They will be dropped
* in mb_unmapped_free_mext().
*/
if (*refcnt == 1)
*refcnt += ref_inc;
else
atomic_add_int(refcnt, ref_inc);
}
m_free(m);
return (top);
fail:
if (ref_inc != 0) {
/*
* Obtain an additional reference on the old mbuf for
* each created EXT_SFBUF mbuf. They will be
* immediately dropped when these mbufs are freed
* below.
*/
if (*refcnt == 1)
*refcnt += ref_inc;
else
atomic_add_int(refcnt, ref_inc);
}
m_free(m);
m_freem(top);
return (NULL);
}
struct mbuf *
mb_unmapped_to_ext(struct mbuf *top)
{
struct mbuf *m, *next, *prev = NULL;
prev = NULL;
for (m = top; m != NULL; m = next) {
/* m might be freed, so cache the next pointer. */
next = m->m_next;
if (m->m_flags & M_EXTPG) {
if (prev != NULL) {
/*
* Remove 'm' from the new chain so
* that the 'top' chain terminates
* before 'm' in case 'top' is freed
* due to an error.
*/
prev->m_next = NULL;
}
m = _mb_unmapped_to_ext(m);
if (m == NULL) {
m_freem(top);
m_freem(next);
return (NULL);
}
if (prev == NULL) {
top = m;
} else {
prev->m_next = m;
}
/*
* Replaced one mbuf with a chain, so we must
* find the end of chain.
*/
prev = m_last(m);
} else {
if (prev != NULL) {
prev->m_next = m;
}
prev = m;
}
}
return (top);
}
/*
* Allocate an empty M_EXTPG mbuf. The ext_free routine is
* responsible for freeing any pages backing this mbuf when it is
* freed.
*/
struct mbuf *
mb_alloc_ext_pgs(int how, m_ext_free_t ext_free)
{
struct mbuf *m;
m = m_get(how, MT_DATA);
if (m == NULL)
return (NULL);
m->m_epg_npgs = 0;
m->m_epg_nrdy = 0;
m->m_epg_1st_off = 0;
m->m_epg_last_len = 0;
m->m_epg_flags = 0;
m->m_epg_hdrlen = 0;
m->m_epg_trllen = 0;
m->m_epg_tls = NULL;
m->m_epg_so = NULL;
m->m_data = NULL;
m->m_flags |= (M_EXT | M_RDONLY | M_EXTPG);
m->m_ext.ext_flags = EXT_FLAG_EMBREF;
m->m_ext.ext_count = 1;
m->m_ext.ext_size = 0;
m->m_ext.ext_free = ext_free;
return (m);
}
/*
* Clean up after mbufs with M_EXT storage attached to them if the
* reference count hits 1.
*/
void
mb_free_ext(struct mbuf *m)
{
volatile u_int *refcnt;
struct mbuf *mref;
int freembuf;
KASSERT(m->m_flags & M_EXT, ("%s: M_EXT not set on %p", __func__, m));
/* See if this is the mbuf that holds the embedded refcount. */
if (m->m_ext.ext_flags & EXT_FLAG_EMBREF) {
refcnt = &m->m_ext.ext_count;
mref = m;
} else {
KASSERT(m->m_ext.ext_cnt != NULL,
("%s: no refcounting pointer on %p", __func__, m));
refcnt = m->m_ext.ext_cnt;
mref = __containerof(refcnt, struct mbuf, m_ext.ext_count);
}
/*
* Check if the header is embedded in the cluster. It is
* important that we can't touch any of the mbuf fields
* after we have freed the external storage, since mbuf
* could have been embedded in it. For now, the mbufs
* embedded into the cluster are always of type EXT_EXTREF,
* and for this type we won't free the mref.
*/
if (m->m_flags & M_NOFREE) {
freembuf = 0;
KASSERT(m->m_ext.ext_type == EXT_EXTREF ||
m->m_ext.ext_type == EXT_RXRING,
("%s: no-free mbuf %p has wrong type", __func__, m));
} else
freembuf = 1;
/* Free attached storage if this mbuf is the only reference to it. */
if (*refcnt == 1 || atomic_fetchadd_int(refcnt, -1) == 1) {
switch (m->m_ext.ext_type) {
case EXT_PACKET:
/* The packet zone is special. */
if (*refcnt == 0)
*refcnt = 1;
uma_zfree(zone_pack, mref);
break;
case EXT_CLUSTER:
uma_zfree(zone_clust, m->m_ext.ext_buf);
m_free_raw(mref);
break;
case EXT_JUMBOP:
uma_zfree(zone_jumbop, m->m_ext.ext_buf);
m_free_raw(mref);
break;
case EXT_JUMBO9:
uma_zfree(zone_jumbo9, m->m_ext.ext_buf);
m_free_raw(mref);
break;
case EXT_JUMBO16:
uma_zfree(zone_jumbo16, m->m_ext.ext_buf);
m_free_raw(mref);
break;
case EXT_SFBUF:
case EXT_NET_DRV:
case EXT_MOD_TYPE:
case EXT_DISPOSABLE:
KASSERT(mref->m_ext.ext_free != NULL,
("%s: ext_free not set", __func__));
mref->m_ext.ext_free(mref);
m_free_raw(mref);
break;
case EXT_EXTREF:
KASSERT(m->m_ext.ext_free != NULL,
("%s: ext_free not set", __func__));
m->m_ext.ext_free(m);
break;
case EXT_RXRING:
KASSERT(m->m_ext.ext_free == NULL,
("%s: ext_free is set", __func__));
break;
default:
KASSERT(m->m_ext.ext_type == 0,
("%s: unknown ext_type", __func__));
}
}
if (freembuf && m != mref)
m_free_raw(m);
}
/*
* Clean up after mbufs with M_EXTPG storage attached to them if the
* reference count hits 1.
*/
void
mb_free_extpg(struct mbuf *m)
{
volatile u_int *refcnt;
struct mbuf *mref;
M_ASSERTEXTPG(m);
/* See if this is the mbuf that holds the embedded refcount. */
if (m->m_ext.ext_flags & EXT_FLAG_EMBREF) {
refcnt = &m->m_ext.ext_count;
mref = m;
} else {
KASSERT(m->m_ext.ext_cnt != NULL,
("%s: no refcounting pointer on %p", __func__, m));
refcnt = m->m_ext.ext_cnt;
mref = __containerof(refcnt, struct mbuf, m_ext.ext_count);
}
/* Free attached storage if this mbuf is the only reference to it. */
if (*refcnt == 1 || atomic_fetchadd_int(refcnt, -1) == 1) {
KASSERT(mref->m_ext.ext_free != NULL,
("%s: ext_free not set", __func__));
mref->m_ext.ext_free(mref);
#ifdef KERN_TLS
if (mref->m_epg_tls != NULL &&
!refcount_release_if_not_last(&mref->m_epg_tls->refcount))
ktls_enqueue_to_free(mref);
else
#endif
m_free_raw(mref);
}
if (m != mref)
m_free_raw(m);
}
/*
* Official mbuf(9) allocation KPI for stack and drivers:
*
* m_get() - a single mbuf without any attachments, sys/mbuf.h.
* m_gethdr() - a single mbuf initialized as M_PKTHDR, sys/mbuf.h.
* m_getcl() - an mbuf + 2k cluster, sys/mbuf.h.
* m_clget() - attach cluster to already allocated mbuf.
* m_cljget() - attach jumbo cluster to already allocated mbuf.
* m_get2() - allocate minimum mbuf that would fit size argument.
* m_getm2() - allocate a chain of mbufs/clusters.
* m_extadd() - attach external cluster to mbuf.
*
* m_free() - free single mbuf with its tags and ext, sys/mbuf.h.
* m_freem() - free chain of mbufs.
*/
int
m_clget(struct mbuf *m, int how)
{
KASSERT((m->m_flags & M_EXT) == 0, ("%s: mbuf %p has M_EXT",
__func__, m));
m->m_ext.ext_buf = (char *)NULL;
uma_zalloc_arg(zone_clust, m, how);
/*
* On a cluster allocation failure, drain the packet zone and retry,
* we might be able to loosen a few clusters up on the drain.
*/
if ((how & M_NOWAIT) && (m->m_ext.ext_buf == NULL)) {
uma_zone_reclaim(zone_pack, UMA_RECLAIM_DRAIN);
uma_zalloc_arg(zone_clust, m, how);
}
MBUF_PROBE2(m__clget, m, how);
return (m->m_flags & M_EXT);
}
/*
* m_cljget() is different from m_clget() as it can allocate clusters without
* attaching them to an mbuf. In that case the return value is the pointer
* to the cluster of the requested size. If an mbuf was specified, it gets
* the cluster attached to it and the return value can be safely ignored.
* For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES.
*/
void *
m_cljget(struct mbuf *m, int how, int size)
{
uma_zone_t zone;
void *retval;
if (m != NULL) {
KASSERT((m->m_flags & M_EXT) == 0, ("%s: mbuf %p has M_EXT",
__func__, m));
m->m_ext.ext_buf = NULL;
}
zone = m_getzone(size);
retval = uma_zalloc_arg(zone, m, how);
MBUF_PROBE4(m__cljget, m, how, size, retval);
return (retval);
}
/*
* m_get2() allocates minimum mbuf that would fit "size" argument.
*/
struct mbuf *
m_get2(int size, int how, short type, int flags)
{
struct mb_args args;
struct mbuf *m, *n;
args.flags = flags;
args.type = type;
if (size <= MHLEN || (size <= MLEN && (flags & M_PKTHDR) == 0))
return (uma_zalloc_arg(zone_mbuf, &args, how));
if (size <= MCLBYTES)
return (uma_zalloc_arg(zone_pack, &args, how));
if (size > MJUMPAGESIZE)
return (NULL);
m = uma_zalloc_arg(zone_mbuf, &args, how);
if (m == NULL)
return (NULL);
n = uma_zalloc_arg(zone_jumbop, m, how);
if (n == NULL) {
m_free_raw(m);
return (NULL);
}
return (m);
}
/*
* m_get3() allocates minimum mbuf that would fit "size" argument.
* Unlike m_get2() it can allocate clusters up to MJUM16BYTES.
*/
struct mbuf *
m_get3(int size, int how, short type, int flags)
{
struct mb_args args;
struct mbuf *m, *n;
uma_zone_t zone;
if (size <= MJUMPAGESIZE)
return (m_get2(size, how, type, flags));
if (size > MJUM16BYTES)
return (NULL);
args.flags = flags;
args.type = type;
m = uma_zalloc_arg(zone_mbuf, &args, how);
if (m == NULL)
return (NULL);
if (size <= MJUM9BYTES)
zone = zone_jumbo9;
else
zone = zone_jumbo16;
n = uma_zalloc_arg(zone, m, how);
if (n == NULL) {
m_free_raw(m);
return (NULL);
}
return (m);
}
/*
* m_getjcl() returns an mbuf with a cluster of the specified size attached.
* For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES.
*/
struct mbuf *
m_getjcl(int how, short type, int flags, int size)
{
struct mb_args args;
struct mbuf *m, *n;
uma_zone_t zone;
if (size == MCLBYTES)
return m_getcl(how, type, flags);
args.flags = flags;
args.type = type;
m = uma_zalloc_arg(zone_mbuf, &args, how);
if (m == NULL)
return (NULL);
zone = m_getzone(size);
n = uma_zalloc_arg(zone, m, how);
if (n == NULL) {
m_free_raw(m);
return (NULL);
}
MBUF_PROBE5(m__getjcl, how, type, flags, size, m);
return (m);
}
/*
* Allocate a given length worth of mbufs and/or clusters (whatever fits
* best) and return a pointer to the top of the allocated chain. If an
* existing mbuf chain is provided, then we will append the new chain
* to the existing one and return a pointer to the provided mbuf.
*/
struct mbuf *
m_getm2(struct mbuf *m, int len, int how, short type, int flags)
{
struct mbuf *mb, *nm = NULL, *mtail = NULL;
KASSERT(len >= 0, ("%s: len is < 0", __func__));
/* Validate flags. */
flags &= (M_PKTHDR | M_EOR);
/* Packet header mbuf must be first in chain. */
if ((flags & M_PKTHDR) && m != NULL)
flags &= ~M_PKTHDR;
/* Loop and append maximum sized mbufs to the chain tail. */
while (len > 0) {
mb = NULL;
if (len > MCLBYTES) {
mb = m_getjcl(M_NOWAIT, type, (flags & M_PKTHDR),
MJUMPAGESIZE);
}
if (mb == NULL) {
if (len >= MINCLSIZE)
mb = m_getcl(how, type, (flags & M_PKTHDR));
else if (flags & M_PKTHDR)
mb = m_gethdr(how, type);
else
mb = m_get(how, type);
/*
* Fail the whole operation if one mbuf can't be
* allocated.
*/
if (mb == NULL) {
m_freem(nm);
return (NULL);
}
}
/* Book keeping. */
len -= M_SIZE(mb);
if (mtail != NULL)
mtail->m_next = mb;
else
nm = mb;
mtail = mb;
flags &= ~M_PKTHDR; /* Only valid on the first mbuf. */
}
if (flags & M_EOR)
mtail->m_flags |= M_EOR; /* Only valid on the last mbuf. */
/* If mbuf was supplied, append new chain to the end of it. */
if (m != NULL) {
for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next)
;
mtail->m_next = nm;
mtail->m_flags &= ~M_EOR;
} else
m = nm;
return (m);
}
/*-
* Configure a provided mbuf to refer to the provided external storage
* buffer and setup a reference count for said buffer.
*
* Arguments:
* mb The existing mbuf to which to attach the provided buffer.
* buf The address of the provided external storage buffer.
* size The size of the provided buffer.
* freef A pointer to a routine that is responsible for freeing the
* provided external storage buffer.
* args A pointer to an argument structure (of any type) to be passed
* to the provided freef routine (may be NULL).
* flags Any other flags to be passed to the provided mbuf.
* type The type that the external storage buffer should be
* labeled with.
*
* Returns:
* Nothing.
*/
void
m_extadd(struct mbuf *mb, char *buf, u_int size, m_ext_free_t freef,
void *arg1, void *arg2, int flags, int type)
{
KASSERT(type != EXT_CLUSTER, ("%s: EXT_CLUSTER not allowed", __func__));
mb->m_flags |= (M_EXT | flags);
mb->m_ext.ext_buf = buf;
mb->m_data = mb->m_ext.ext_buf;
mb->m_ext.ext_size = size;
mb->m_ext.ext_free = freef;
mb->m_ext.ext_arg1 = arg1;
mb->m_ext.ext_arg2 = arg2;
mb->m_ext.ext_type = type;
if (type != EXT_EXTREF) {
mb->m_ext.ext_count = 1;
mb->m_ext.ext_flags = EXT_FLAG_EMBREF;
} else
mb->m_ext.ext_flags = 0;
}
/*
* Free an entire chain of mbufs and associated external buffers, if
* applicable.
*/
void
m_freem(struct mbuf *mb)
{
MBUF_PROBE1(m__freem, mb);
while (mb != NULL)
mb = m_free(mb);
}
/*
* Temporary primitive to allow freeing without going through m_free.
*/
void
m_free_raw(struct mbuf *mb)
{
uma_zfree(zone_mbuf, mb);
}
int
m_snd_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
struct m_snd_tag **mstp)
{
if (ifp->if_snd_tag_alloc == NULL)
return (EOPNOTSUPP);
return (ifp->if_snd_tag_alloc(ifp, params, mstp));
}
void
m_snd_tag_init(struct m_snd_tag *mst, struct ifnet *ifp,
const struct if_snd_tag_sw *sw)
{
if_ref(ifp);
mst->ifp = ifp;
refcount_init(&mst->refcount, 1);
mst->sw = sw;
counter_u64_add(snd_tag_count, 1);
}
void
m_snd_tag_destroy(struct m_snd_tag *mst)
{
struct ifnet *ifp;
ifp = mst->ifp;
mst->sw->snd_tag_free(mst);
if_rele(ifp);
counter_u64_add(snd_tag_count, -1);
}
-void
-m_rcvif_serialize(struct mbuf *m)
-{
- u_short idx, gen;
-
- M_ASSERTPKTHDR(m);
- idx = m->m_pkthdr.rcvif->if_index;
- gen = m->m_pkthdr.rcvif->if_idxgen;
- m->m_pkthdr.rcvidx = idx;
- m->m_pkthdr.rcvgen = gen;
-}
-
-struct ifnet *
-m_rcvif_restore(struct mbuf *m)
-{
- struct ifnet *ifp;
-
- M_ASSERTPKTHDR(m);
- NET_EPOCH_ASSERT();
-
- ifp = ifnet_byindexgen(m->m_pkthdr.rcvidx, m->m_pkthdr.rcvgen);
- if (ifp == NULL || (ifp->if_flags & IFF_DYING))
- return (NULL);
-
- return (m->m_pkthdr.rcvif = ifp);
-}
-
/*
* Allocate an mbuf with anonymous external pages.
*/
struct mbuf *
mb_alloc_ext_plus_pages(int len, int how)
{
struct mbuf *m;
vm_page_t pg;
int i, npgs;
m = mb_alloc_ext_pgs(how, mb_free_mext_pgs);
if (m == NULL)
return (NULL);
m->m_epg_flags |= EPG_FLAG_ANON;
npgs = howmany(len, PAGE_SIZE);
for (i = 0; i < npgs; i++) {
do {
pg = vm_page_alloc_noobj(VM_ALLOC_NODUMP |
VM_ALLOC_WIRED);
if (pg == NULL) {
if (how == M_NOWAIT) {
m->m_epg_npgs = i;
m_free(m);
return (NULL);
}
vm_wait(NULL);
}
} while (pg == NULL);
m->m_epg_pa[i] = VM_PAGE_TO_PHYS(pg);
}
m->m_epg_npgs = npgs;
return (m);
}
/*
* Copy the data in the mbuf chain to a chain of mbufs with anonymous external
* unmapped pages.
* len is the length of data in the input mbuf chain.
* mlen is the maximum number of bytes put into each ext_page mbuf.
*/
struct mbuf *
mb_mapped_to_unmapped(struct mbuf *mp, int len, int mlen, int how,
struct mbuf **mlast)
{
struct mbuf *m, *mout;
char *pgpos, *mbpos;
int i, mblen, mbufsiz, pglen, xfer;
if (len == 0)
return (NULL);
mbufsiz = min(mlen, len);
m = mout = mb_alloc_ext_plus_pages(mbufsiz, how);
if (m == NULL)
return (m);
pgpos = (char *)(void *)PHYS_TO_DMAP(m->m_epg_pa[0]);
pglen = PAGE_SIZE;
mblen = 0;
i = 0;
do {
if (pglen == 0) {
if (++i == m->m_epg_npgs) {
m->m_epg_last_len = PAGE_SIZE;
mbufsiz = min(mlen, len);
m->m_next = mb_alloc_ext_plus_pages(mbufsiz,
how);
m = m->m_next;
if (m == NULL) {
m_freem(mout);
return (m);
}
i = 0;
}
pgpos = (char *)(void *)PHYS_TO_DMAP(m->m_epg_pa[i]);
pglen = PAGE_SIZE;
}
while (mblen == 0) {
if (mp == NULL) {
m_freem(mout);
return (NULL);
}
KASSERT((mp->m_flags & M_EXTPG) == 0,
("mb_copym_ext_pgs: ext_pgs input mbuf"));
mbpos = mtod(mp, char *);
mblen = mp->m_len;
mp = mp->m_next;
}
xfer = min(mblen, pglen);
memcpy(pgpos, mbpos, xfer);
pgpos += xfer;
mbpos += xfer;
pglen -= xfer;
mblen -= xfer;
len -= xfer;
m->m_len += xfer;
} while (len > 0);
m->m_epg_last_len = PAGE_SIZE - pglen;
if (mlast != NULL)
*mlast = m;
return (mout);
}
diff --git a/sys/net/if.c b/sys/net/if.c
index ff7071cea364..3b303fe42e99 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -1,4584 +1,4561 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 2010 Bjoern A. Zeeb <bz@FreeBSD.org>
* Copyright (c) 1980, 1986, 1993
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)if.c 8.5 (Berkeley) 1/9/95
* $FreeBSD$
*/
#include "opt_bpf.h"
#include "opt_inet6.h"
#include "opt_inet.h"
#include "opt_ddb.h"
#include <sys/param.h>
#include <sys/capsicum.h>
#include <sys/conf.h>
#include <sys/eventhandler.h>
#include <sys/malloc.h>
#include <sys/domainset.h>
#include <sys/sbuf.h>
#include <sys/bus.h>
#include <sys/epoch.h>
#include <sys/mbuf.h>
#include <sys/systm.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/protosw.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/refcount.h>
#include <sys/module.h>
#include <sys/rwlock.h>
#include <sys/sockio.h>
#include <sys/syslog.h>
#include <sys/sysctl.h>
#include <sys/sysent.h>
#include <sys/taskqueue.h>
#include <sys/domain.h>
#include <sys/jail.h>
#include <sys/priv.h>
#ifdef DDB
#include <ddb/ddb.h>
#endif
#include <machine/stdarg.h>
#include <vm/uma.h>
#include <net/bpf.h>
#include <net/ethernet.h>
#include <net/if.h>
#include <net/if_arp.h>
#include <net/if_clone.h>
#include <net/if_dl.h>
#include <net/if_types.h>
#include <net/if_var.h>
#include <net/if_media.h>
#include <net/if_mib.h>
#include <net/if_vlan_var.h>
#include <net/radix.h>
#include <net/route.h>
#include <net/route/route_ctl.h>
#include <net/vnet.h>
#if defined(INET) || defined(INET6)
#include <net/ethernet.h>
#include <netinet/in.h>
#include <netinet/in_var.h>
#include <netinet/ip.h>
#include <netinet/ip_carp.h>
#ifdef INET
#include <net/debugnet.h>
#include <netinet/if_ether.h>
#endif /* INET */
#ifdef INET6
#include <netinet6/in6_var.h>
#include <netinet6/in6_ifattach.h>
#endif /* INET6 */
#endif /* INET || INET6 */
#include <security/mac/mac_framework.h>
/*
* Consumers of struct ifreq such as tcpdump assume no pad between ifr_name
* and ifr_ifru when it is used in SIOCGIFCONF.
*/
_Static_assert(sizeof(((struct ifreq *)0)->ifr_name) ==
offsetof(struct ifreq, ifr_ifru), "gap between ifr_name and ifr_ifru");
__read_mostly epoch_t net_epoch_preempt;
#ifdef COMPAT_FREEBSD32
#include <sys/mount.h>
#include <compat/freebsd32/freebsd32.h>
struct ifreq_buffer32 {
uint32_t length; /* (size_t) */
uint32_t buffer; /* (void *) */
};
/*
* Interface request structure used for socket
* ioctl's. All interface ioctl's must have parameter
* definitions which begin with ifr_name. The
* remainder may be interface specific.
*/
struct ifreq32 {
char ifr_name[IFNAMSIZ]; /* if name, e.g. "en0" */
union {
struct sockaddr ifru_addr;
struct sockaddr ifru_dstaddr;
struct sockaddr ifru_broadaddr;
struct ifreq_buffer32 ifru_buffer;
short ifru_flags[2];
short ifru_index;
int ifru_jid;
int ifru_metric;
int ifru_mtu;
int ifru_phys;
int ifru_media;
uint32_t ifru_data;
int ifru_cap[2];
u_int ifru_fib;
u_char ifru_vlan_pcp;
} ifr_ifru;
};
CTASSERT(sizeof(struct ifreq) == sizeof(struct ifreq32));
CTASSERT(__offsetof(struct ifreq, ifr_ifru) ==
__offsetof(struct ifreq32, ifr_ifru));
struct ifconf32 {
int32_t ifc_len;
union {
uint32_t ifcu_buf;
uint32_t ifcu_req;
} ifc_ifcu;
};
#define SIOCGIFCONF32 _IOWR('i', 36, struct ifconf32)
struct ifdrv32 {
char ifd_name[IFNAMSIZ];
uint32_t ifd_cmd;
uint32_t ifd_len;
uint32_t ifd_data;
};
#define SIOCSDRVSPEC32 _IOC_NEWTYPE(SIOCSDRVSPEC, struct ifdrv32)
#define SIOCGDRVSPEC32 _IOC_NEWTYPE(SIOCGDRVSPEC, struct ifdrv32)
struct ifgroupreq32 {
char ifgr_name[IFNAMSIZ];
u_int ifgr_len;
union {
char ifgru_group[IFNAMSIZ];
uint32_t ifgru_groups;
} ifgr_ifgru;
};
#define SIOCAIFGROUP32 _IOC_NEWTYPE(SIOCAIFGROUP, struct ifgroupreq32)
#define SIOCGIFGROUP32 _IOC_NEWTYPE(SIOCGIFGROUP, struct ifgroupreq32)
#define SIOCDIFGROUP32 _IOC_NEWTYPE(SIOCDIFGROUP, struct ifgroupreq32)
#define SIOCGIFGMEMB32 _IOC_NEWTYPE(SIOCGIFGMEMB, struct ifgroupreq32)
struct ifmediareq32 {
char ifm_name[IFNAMSIZ];
int ifm_current;
int ifm_mask;
int ifm_status;
int ifm_active;
int ifm_count;
uint32_t ifm_ulist; /* (int *) */
};
#define SIOCGIFMEDIA32 _IOC_NEWTYPE(SIOCGIFMEDIA, struct ifmediareq32)
#define SIOCGIFXMEDIA32 _IOC_NEWTYPE(SIOCGIFXMEDIA, struct ifmediareq32)
#endif /* COMPAT_FREEBSD32 */
union ifreq_union {
struct ifreq ifr;
#ifdef COMPAT_FREEBSD32
struct ifreq32 ifr32;
#endif
};
SYSCTL_NODE(_net, PF_LINK, link, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
"Link layers");
SYSCTL_NODE(_net_link, 0, generic, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
"Generic link-management");
SYSCTL_INT(_net_link, OID_AUTO, ifqmaxlen, CTLFLAG_RDTUN,
&ifqmaxlen, 0, "max send queue size");
/* Log link state change events */
static int log_link_state_change = 1;
SYSCTL_INT(_net_link, OID_AUTO, log_link_state_change, CTLFLAG_RW,
&log_link_state_change, 0,
"log interface link state change events");
/* Log promiscuous mode change events */
static int log_promisc_mode_change = 1;
SYSCTL_INT(_net_link, OID_AUTO, log_promisc_mode_change, CTLFLAG_RDTUN,
&log_promisc_mode_change, 1,
"log promiscuous mode change events");
/* Interface description */
static unsigned int ifdescr_maxlen = 1024;
SYSCTL_UINT(_net, OID_AUTO, ifdescr_maxlen, CTLFLAG_RW,
&ifdescr_maxlen, 0,
"administrative maximum length for interface description");
static MALLOC_DEFINE(M_IFDESCR, "ifdescr", "ifnet descriptions");
/* global sx for non-critical path ifdescr */
static struct sx ifdescr_sx;
SX_SYSINIT(ifdescr_sx, &ifdescr_sx, "ifnet descr");
void (*ng_ether_link_state_p)(struct ifnet *ifp, int state);
void (*lagg_linkstate_p)(struct ifnet *ifp, int state);
/* These are external hooks for CARP. */
void (*carp_linkstate_p)(struct ifnet *ifp);
void (*carp_demote_adj_p)(int, char *);
int (*carp_master_p)(struct ifaddr *);
#if defined(INET) || defined(INET6)
int (*carp_forus_p)(struct ifnet *ifp, u_char *dhost);
int (*carp_output_p)(struct ifnet *ifp, struct mbuf *m,
const struct sockaddr *sa);
int (*carp_ioctl_p)(struct ifreq *, u_long, struct thread *);
int (*carp_attach_p)(struct ifaddr *, int);
void (*carp_detach_p)(struct ifaddr *, bool);
#endif
#ifdef INET
int (*carp_iamatch_p)(struct ifaddr *, uint8_t **);
#endif
#ifdef INET6
struct ifaddr *(*carp_iamatch6_p)(struct ifnet *ifp, struct in6_addr *taddr6);
caddr_t (*carp_macmatch6_p)(struct ifnet *ifp, struct mbuf *m,
const struct in6_addr *taddr);
#endif
struct mbuf *(*tbr_dequeue_ptr)(struct ifaltq *, int) = NULL;
/*
* XXX: Style; these should be sorted alphabetically, and unprototyped
* static functions should be prototyped. Currently they are sorted by
* declaration order.
*/
static void if_attachdomain(void *);
static void if_attachdomain1(struct ifnet *);
static int ifconf(u_long, caddr_t);
static void if_input_default(struct ifnet *, struct mbuf *);
static int if_requestencap_default(struct ifnet *, struct if_encap_req *);
static void if_route(struct ifnet *, int flag, int fam);
static int if_setflag(struct ifnet *, int, int, int *, int);
static int if_transmit(struct ifnet *ifp, struct mbuf *m);
static void if_unroute(struct ifnet *, int flag, int fam);
static int if_delmulti_locked(struct ifnet *, struct ifmultiaddr *, int);
static void do_link_state_change(void *, int);
static int if_getgroup(struct ifgroupreq *, struct ifnet *);
static int if_getgroupmembers(struct ifgroupreq *);
static void if_delgroups(struct ifnet *);
static void if_attach_internal(struct ifnet *, bool);
static int if_detach_internal(struct ifnet *, bool);
static void if_siocaddmulti(void *, int);
static void if_link_ifnet(struct ifnet *);
static bool if_unlink_ifnet(struct ifnet *, bool);
#ifdef VIMAGE
static int if_vmove(struct ifnet *, struct vnet *);
#endif
#ifdef INET6
/*
* XXX: declare here to avoid to include many inet6 related files..
* should be more generalized?
*/
extern void nd6_setmtu(struct ifnet *);
#endif
/* ipsec helper hooks */
VNET_DEFINE(struct hhook_head *, ipsec_hhh_in[HHOOK_IPSEC_COUNT]);
VNET_DEFINE(struct hhook_head *, ipsec_hhh_out[HHOOK_IPSEC_COUNT]);
int ifqmaxlen = IFQ_MAXLEN;
VNET_DEFINE(struct ifnethead, ifnet); /* depend on static init XXX */
VNET_DEFINE(struct ifgrouphead, ifg_head);
/* Table of ifnet by index. */
static int if_index;
static int if_indexlim = 8;
-static struct ifindex_entry {
- struct ifnet *ife_ifnet;
- uint16_t ife_gencnt;
-} *ifindex_table;
+static struct ifnet **ifindex_table;
SYSCTL_NODE(_net_link_generic, IFMIB_SYSTEM, system,
CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
"Variables global to all interfaces");
static int
sysctl_ifcount(SYSCTL_HANDLER_ARGS)
{
int rv = 0;
IFNET_RLOCK();
for (int i = 1; i <= if_index; i++)
- if (ifindex_table[i].ife_ifnet != NULL &&
- ifindex_table[i].ife_ifnet->if_vnet == curvnet)
+ if (ifindex_table[i] != NULL &&
+ ifindex_table[i]->if_vnet == curvnet)
rv = i;
IFNET_RUNLOCK();
return (sysctl_handle_int(oidp, &rv, 0, req));
}
SYSCTL_PROC(_net_link_generic_system, IFMIB_IFCOUNT, ifcount,
CTLTYPE_INT | CTLFLAG_VNET | CTLFLAG_RD, NULL, 0, sysctl_ifcount, "I",
"Maximum known interface index");
/*
* The global network interface list (V_ifnet) and related state (such as
* if_index, if_indexlim, and ifindex_table) are protected by an sxlock.
* This may be acquired to stabilise the list, or we may rely on NET_EPOCH.
*/
struct sx ifnet_sxlock;
SX_SYSINIT_FLAGS(ifnet_sx, &ifnet_sxlock, "ifnet_sx", SX_RECURSE);
struct sx ifnet_detach_sxlock;
SX_SYSINIT_FLAGS(ifnet_detach, &ifnet_detach_sxlock, "ifnet_detach_sx",
SX_RECURSE);
#ifdef VIMAGE
#define VNET_IS_SHUTTING_DOWN(_vnet) \
((_vnet)->vnet_shutdown && (_vnet)->vnet_state < SI_SUB_VNET_DONE)
#endif
static if_com_alloc_t *if_com_alloc[256];
static if_com_free_t *if_com_free[256];
static MALLOC_DEFINE(M_IFNET, "ifnet", "interface internals");
MALLOC_DEFINE(M_IFADDR, "ifaddr", "interface address");
MALLOC_DEFINE(M_IFMADDR, "ether_multi", "link-level multicast address");
struct ifnet *
ifnet_byindex(u_int idx)
{
struct ifnet *ifp;
NET_EPOCH_ASSERT();
if (__predict_false(idx > if_index))
return (NULL);
- ifp = ck_pr_load_ptr(&ifindex_table[idx].ife_ifnet);
+ ifp = ck_pr_load_ptr(&ifindex_table[idx]);
if (curvnet != NULL && ifp != NULL && ifp->if_vnet != curvnet)
ifp = NULL;
return (ifp);
}
struct ifnet *
ifnet_byindex_ref(u_int idx)
{
struct ifnet *ifp;
ifp = ifnet_byindex(idx);
if (ifp == NULL || (ifp->if_flags & IFF_DYING))
return (NULL);
if (!if_try_ref(ifp))
return (NULL);
return (ifp);
}
-struct ifnet *
-ifnet_byindexgen(uint16_t idx, uint16_t gen)
-{
- struct ifnet *ifp;
-
- NET_EPOCH_ASSERT();
-
- if (__predict_false(idx > if_index))
- return (NULL);
-
- ifp = ck_pr_load_ptr(&ifindex_table[idx].ife_ifnet);
-
- if (ifindex_table[idx].ife_gencnt == gen)
- return (ifp);
- else
- return (NULL);
-}
-
/*
* Network interface utility routines.
*
* Routines with ifa_ifwith* names take sockaddr *'s as
* parameters.
*/
static void
if_init(void *arg __unused)
{
ifindex_table = malloc(if_indexlim * sizeof(*ifindex_table),
M_IFNET, M_WAITOK | M_ZERO);
}
SYSINIT(if_init, SI_SUB_INIT_IF, SI_ORDER_SECOND, if_init, NULL);
static void
vnet_if_init(const void *unused __unused)
{
CK_STAILQ_INIT(&V_ifnet);
CK_STAILQ_INIT(&V_ifg_head);
vnet_if_clone_init();
}
VNET_SYSINIT(vnet_if_init, SI_SUB_INIT_IF, SI_ORDER_SECOND, vnet_if_init,
NULL);
static void
if_link_ifnet(struct ifnet *ifp)
{
IFNET_WLOCK();
CK_STAILQ_INSERT_TAIL(&V_ifnet, ifp, if_link);
#ifdef VIMAGE
curvnet->vnet_ifcnt++;
#endif
IFNET_WUNLOCK();
}
static bool
if_unlink_ifnet(struct ifnet *ifp, bool vmove)
{
struct ifnet *iter;
int found = 0;
IFNET_WLOCK();
CK_STAILQ_FOREACH(iter, &V_ifnet, if_link)
if (iter == ifp) {
CK_STAILQ_REMOVE(&V_ifnet, ifp, ifnet, if_link);
if (!vmove)
ifp->if_flags |= IFF_DYING;
found = 1;
break;
}
#ifdef VIMAGE
curvnet->vnet_ifcnt--;
#endif
IFNET_WUNLOCK();
return (found);
}
#ifdef VIMAGE
static void
vnet_if_return(const void *unused __unused)
{
struct ifnet *ifp, *nifp;
struct ifnet **pending;
int found __diagused;
int i;
i = 0;
/*
* We need to protect our access to the V_ifnet tailq. Ordinarily we'd
* enter NET_EPOCH, but that's not possible, because if_vmove() calls
* if_detach_internal(), which waits for NET_EPOCH callbacks to
* complete. We can't do that from within NET_EPOCH.
*
* However, we can also use the IFNET_xLOCK, which is the V_ifnet
* read/write lock. We cannot hold the lock as we call if_vmove()
* though, as that presents LOR w.r.t ifnet_sx, in_multi_sx and iflib
* ctx lock.
*/
IFNET_WLOCK();
pending = malloc(sizeof(struct ifnet *) * curvnet->vnet_ifcnt,
M_IFNET, M_WAITOK | M_ZERO);
/* Return all inherited interfaces to their parent vnets. */
CK_STAILQ_FOREACH_SAFE(ifp, &V_ifnet, if_link, nifp) {
if (ifp->if_home_vnet != ifp->if_vnet) {
found = if_unlink_ifnet(ifp, true);
MPASS(found);
pending[i++] = ifp;
}
}
IFNET_WUNLOCK();
for (int j = 0; j < i; j++) {
if_vmove(pending[j], pending[j]->if_home_vnet);
}
free(pending, M_IFNET);
}
VNET_SYSUNINIT(vnet_if_return, SI_SUB_VNET_DONE, SI_ORDER_ANY,
vnet_if_return, NULL);
#endif
/*
* Allocate a struct ifnet and an index for an interface. A layer 2
* common structure will also be allocated if an allocation routine is
* registered for the passed type.
*/
static struct ifnet *
if_alloc_domain(u_char type, int numa_domain)
{
struct ifnet *ifp;
u_short idx;
KASSERT(numa_domain <= IF_NODOM, ("numa_domain too large"));
if (numa_domain == IF_NODOM)
ifp = malloc(sizeof(struct ifnet), M_IFNET,
M_WAITOK | M_ZERO);
else
ifp = malloc_domainset(sizeof(struct ifnet), M_IFNET,
DOMAINSET_PREF(numa_domain), M_WAITOK | M_ZERO);
ifp->if_type = type;
ifp->if_alloctype = type;
ifp->if_numa_domain = numa_domain;
#ifdef VIMAGE
ifp->if_vnet = curvnet;
#endif
if (if_com_alloc[type] != NULL) {
ifp->if_l2com = if_com_alloc[type](type, ifp);
KASSERT(ifp->if_l2com, ("%s: if_com_alloc[%u] failed", __func__,
type));
}
IF_ADDR_LOCK_INIT(ifp);
TASK_INIT(&ifp->if_linktask, 0, do_link_state_change, ifp);
TASK_INIT(&ifp->if_addmultitask, 0, if_siocaddmulti, ifp);
ifp->if_afdata_initialized = 0;
IF_AFDATA_LOCK_INIT(ifp);
CK_STAILQ_INIT(&ifp->if_addrhead);
CK_STAILQ_INIT(&ifp->if_multiaddrs);
CK_STAILQ_INIT(&ifp->if_groups);
#ifdef MAC
mac_ifnet_init(ifp);
#endif
ifq_init(&ifp->if_snd, ifp);
refcount_init(&ifp->if_refcount, 1); /* Index reference. */
for (int i = 0; i < IFCOUNTERS; i++)
ifp->if_counters[i] = counter_u64_alloc(M_WAITOK);
ifp->if_get_counter = if_get_counter_default;
ifp->if_pcp = IFNET_PCP_NONE;
/* Allocate an ifindex array entry. */
IFNET_WLOCK();
/*
* Try to find an empty slot below if_index. If we fail, take the
* next slot.
*/
for (idx = 1; idx <= if_index; idx++) {
- if (ifindex_table[idx].ife_ifnet == NULL)
+ if (ifindex_table[idx] == NULL)
break;
}
/* Catch if_index overflow. */
if (idx >= if_indexlim) {
- struct ifindex_entry *new, *old;
+ struct ifnet **new, **old;
int newlim;
newlim = if_indexlim * 2;
new = malloc(newlim * sizeof(*new), M_IFNET, M_WAITOK | M_ZERO);
memcpy(new, ifindex_table, if_indexlim * sizeof(*new));
old = ifindex_table;
ck_pr_store_ptr(&ifindex_table, new);
if_indexlim = newlim;
epoch_wait_preempt(net_epoch_preempt);
free(old, M_IFNET);
}
if (idx > if_index)
if_index = idx;
ifp->if_index = idx;
- ifp->if_idxgen = ifindex_table[idx].ife_gencnt;
- ck_pr_store_ptr(&ifindex_table[idx].ife_ifnet, ifp);
+ ck_pr_store_ptr(&ifindex_table[idx], ifp);
IFNET_WUNLOCK();
return (ifp);
}
struct ifnet *
if_alloc_dev(u_char type, device_t dev)
{
int numa_domain;
if (dev == NULL || bus_get_domain(dev, &numa_domain) != 0)
return (if_alloc_domain(type, IF_NODOM));
return (if_alloc_domain(type, numa_domain));
}
struct ifnet *
if_alloc(u_char type)
{
return (if_alloc_domain(type, IF_NODOM));
}
/*
* Do the actual work of freeing a struct ifnet, and layer 2 common
* structure. This call is made when the network epoch guarantees
* us that nobody holds a pointer to the interface.
*/
static void
if_free_deferred(epoch_context_t ctx)
{
struct ifnet *ifp = __containerof(ctx, struct ifnet, if_epoch_ctx);
KASSERT((ifp->if_flags & IFF_DYING),
("%s: interface not dying", __func__));
if (if_com_free[ifp->if_alloctype] != NULL)
if_com_free[ifp->if_alloctype](ifp->if_l2com,
ifp->if_alloctype);
#ifdef MAC
mac_ifnet_destroy(ifp);
#endif /* MAC */
IF_AFDATA_DESTROY(ifp);
IF_ADDR_LOCK_DESTROY(ifp);
ifq_delete(&ifp->if_snd);
for (int i = 0; i < IFCOUNTERS; i++)
counter_u64_free(ifp->if_counters[i]);
free(ifp->if_description, M_IFDESCR);
free(ifp->if_hw_addr, M_IFADDR);
free(ifp, M_IFNET);
}
/*
* Deregister an interface and free the associated storage.
*/
void
if_free(struct ifnet *ifp)
{
ifp->if_flags |= IFF_DYING; /* XXX: Locking */
/*
* XXXGL: An interface index is really an alias to ifp pointer.
* Why would we clear the alias now, and not in the deferred
* context? Indeed there is nothing wrong with some network
* thread obtaining ifp via ifnet_byindex() inside the network
* epoch and then dereferencing ifp while we perform if_free(),
* and after if_free() finished, too.
*
* This early index freeing was important back when ifindex was
* virtualized and interface would outlive the vnet.
*/
IFNET_WLOCK();
- MPASS(ifindex_table[ifp->if_index].ife_ifnet == ifp);
- ck_pr_store_ptr(&ifindex_table[ifp->if_index].ife_ifnet, NULL);
- ifindex_table[ifp->if_index].ife_gencnt++;
- while (if_index > 0 && ifindex_table[if_index].ife_ifnet == NULL)
+ MPASS(ifindex_table[ifp->if_index] == ifp);
+ ck_pr_store_ptr(&ifindex_table[ifp->if_index], NULL);
+ while (if_index > 0 && ifindex_table[if_index] == NULL)
if_index--;
IFNET_WUNLOCK();
if (refcount_release(&ifp->if_refcount))
NET_EPOCH_CALL(if_free_deferred, &ifp->if_epoch_ctx);
}
/*
* Interfaces to keep an ifnet type-stable despite the possibility of the
* driver calling if_free(). If there are additional references, we defer
* freeing the underlying data structure.
*/
void
if_ref(struct ifnet *ifp)
{
u_int old __diagused;
/* We don't assert the ifnet list lock here, but arguably should. */
old = refcount_acquire(&ifp->if_refcount);
KASSERT(old > 0, ("%s: ifp %p has 0 refs", __func__, ifp));
}
bool
if_try_ref(struct ifnet *ifp)
{
NET_EPOCH_ASSERT();
return (refcount_acquire_if_not_zero(&ifp->if_refcount));
}
void
if_rele(struct ifnet *ifp)
{
if (!refcount_release(&ifp->if_refcount))
return;
NET_EPOCH_CALL(if_free_deferred, &ifp->if_epoch_ctx);
}
void
ifq_init(struct ifaltq *ifq, struct ifnet *ifp)
{
mtx_init(&ifq->ifq_mtx, ifp->if_xname, "if send queue", MTX_DEF);
if (ifq->ifq_maxlen == 0)
ifq->ifq_maxlen = ifqmaxlen;
ifq->altq_type = 0;
ifq->altq_disc = NULL;
ifq->altq_flags &= ALTQF_CANTCHANGE;
ifq->altq_tbr = NULL;
ifq->altq_ifp = ifp;
}
void
ifq_delete(struct ifaltq *ifq)
{
mtx_destroy(&ifq->ifq_mtx);
}
/*
* Perform generic interface initialization tasks and attach the interface
* to the list of "active" interfaces. If vmove flag is set on entry
* to if_attach_internal(), perform only a limited subset of initialization
* tasks, given that we are moving from one vnet to another an ifnet which
* has already been fully initialized.
*
* Note that if_detach_internal() removes group membership unconditionally
* even when vmove flag is set, and if_attach_internal() adds only IFG_ALL.
* Thus, when if_vmove() is applied to a cloned interface, group membership
* is lost while a cloned one always joins a group whose name is
* ifc->ifc_name. To recover this after if_detach_internal() and
* if_attach_internal(), the cloner should be specified to
* if_attach_internal() via ifc. If it is non-NULL, if_attach_internal()
* attempts to join a group whose name is ifc->ifc_name.
*
* XXX:
* - The decision to return void and thus require this function to
* succeed is questionable.
* - We should probably do more sanity checking. For instance we don't
* do anything to insure if_xname is unique or non-empty.
*/
void
if_attach(struct ifnet *ifp)
{
if_attach_internal(ifp, false);
}
/*
* Compute the least common TSO limit.
*/
void
if_hw_tsomax_common(if_t ifp, struct ifnet_hw_tsomax *pmax)
{
/*
* 1) If there is no limit currently, take the limit from
* the network adapter.
*
* 2) If the network adapter has a limit below the current
* limit, apply it.
*/
if (pmax->tsomaxbytes == 0 || (ifp->if_hw_tsomax != 0 &&
ifp->if_hw_tsomax < pmax->tsomaxbytes)) {
pmax->tsomaxbytes = ifp->if_hw_tsomax;
}
if (pmax->tsomaxsegcount == 0 || (ifp->if_hw_tsomaxsegcount != 0 &&
ifp->if_hw_tsomaxsegcount < pmax->tsomaxsegcount)) {
pmax->tsomaxsegcount = ifp->if_hw_tsomaxsegcount;
}
if (pmax->tsomaxsegsize == 0 || (ifp->if_hw_tsomaxsegsize != 0 &&
ifp->if_hw_tsomaxsegsize < pmax->tsomaxsegsize)) {
pmax->tsomaxsegsize = ifp->if_hw_tsomaxsegsize;
}
}
/*
* Update TSO limit of a network adapter.
*
* Returns zero if no change. Else non-zero.
*/
int
if_hw_tsomax_update(if_t ifp, struct ifnet_hw_tsomax *pmax)
{
int retval = 0;
if (ifp->if_hw_tsomax != pmax->tsomaxbytes) {
ifp->if_hw_tsomax = pmax->tsomaxbytes;
retval++;
}
if (ifp->if_hw_tsomaxsegsize != pmax->tsomaxsegsize) {
ifp->if_hw_tsomaxsegsize = pmax->tsomaxsegsize;
retval++;
}
if (ifp->if_hw_tsomaxsegcount != pmax->tsomaxsegcount) {
ifp->if_hw_tsomaxsegcount = pmax->tsomaxsegcount;
retval++;
}
return (retval);
}
static void
if_attach_internal(struct ifnet *ifp, bool vmove)
{
unsigned socksize, ifasize;
int namelen, masklen;
struct sockaddr_dl *sdl;
struct ifaddr *ifa;
- MPASS(ifindex_table[ifp->if_index].ife_ifnet == ifp);
+ MPASS(ifindex_table[ifp->if_index] == ifp);
#ifdef VIMAGE
ifp->if_vnet = curvnet;
if (ifp->if_home_vnet == NULL)
ifp->if_home_vnet = curvnet;
#endif
if_addgroup(ifp, IFG_ALL);
#ifdef VIMAGE
/* Restore group membership for cloned interface. */
if (vmove)
if_clone_restoregroup(ifp);
#endif
getmicrotime(&ifp->if_lastchange);
ifp->if_epoch = time_uptime;
KASSERT((ifp->if_transmit == NULL && ifp->if_qflush == NULL) ||
(ifp->if_transmit != NULL && ifp->if_qflush != NULL),
("transmit and qflush must both either be set or both be NULL"));
if (ifp->if_transmit == NULL) {
ifp->if_transmit = if_transmit;
ifp->if_qflush = if_qflush;
}
if (ifp->if_input == NULL)
ifp->if_input = if_input_default;
if (ifp->if_requestencap == NULL)
ifp->if_requestencap = if_requestencap_default;
if (!vmove) {
#ifdef MAC
mac_ifnet_create(ifp);
#endif
/*
* Create a Link Level name for this device.
*/
namelen = strlen(ifp->if_xname);
/*
* Always save enough space for any possiable name so we
* can do a rename in place later.
*/
masklen = offsetof(struct sockaddr_dl, sdl_data[0]) + IFNAMSIZ;
socksize = masklen + ifp->if_addrlen;
if (socksize < sizeof(*sdl))
socksize = sizeof(*sdl);
socksize = roundup2(socksize, sizeof(long));
ifasize = sizeof(*ifa) + 2 * socksize;
ifa = ifa_alloc(ifasize, M_WAITOK);
sdl = (struct sockaddr_dl *)(ifa + 1);
sdl->sdl_len = socksize;
sdl->sdl_family = AF_LINK;
bcopy(ifp->if_xname, sdl->sdl_data, namelen);
sdl->sdl_nlen = namelen;
sdl->sdl_index = ifp->if_index;
sdl->sdl_type = ifp->if_type;
ifp->if_addr = ifa;
ifa->ifa_ifp = ifp;
ifa->ifa_addr = (struct sockaddr *)sdl;
sdl = (struct sockaddr_dl *)(socksize + (caddr_t)sdl);
ifa->ifa_netmask = (struct sockaddr *)sdl;
sdl->sdl_len = masklen;
while (namelen != 0)
sdl->sdl_data[--namelen] = 0xff;
CK_STAILQ_INSERT_HEAD(&ifp->if_addrhead, ifa, ifa_link);
/* Reliably crash if used uninitialized. */
ifp->if_broadcastaddr = NULL;
if (ifp->if_type == IFT_ETHER) {
ifp->if_hw_addr = malloc(ifp->if_addrlen, M_IFADDR,
M_WAITOK | M_ZERO);
}
#if defined(INET) || defined(INET6)
/* Use defaults for TSO, if nothing is set */
if (ifp->if_hw_tsomax == 0 &&
ifp->if_hw_tsomaxsegcount == 0 &&
ifp->if_hw_tsomaxsegsize == 0) {
/*
* The TSO defaults needs to be such that an
* NFS mbuf list of 35 mbufs totalling just
* below 64K works and that a chain of mbufs
* can be defragged into at most 32 segments:
*/
ifp->if_hw_tsomax = min(IP_MAXPACKET, (32 * MCLBYTES) -
(ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN));
ifp->if_hw_tsomaxsegcount = 35;
ifp->if_hw_tsomaxsegsize = 2048; /* 2K */
/* XXX some drivers set IFCAP_TSO after ethernet attach */
if (ifp->if_capabilities & IFCAP_TSO) {
if_printf(ifp, "Using defaults for TSO: %u/%u/%u\n",
ifp->if_hw_tsomax,
ifp->if_hw_tsomaxsegcount,
ifp->if_hw_tsomaxsegsize);
}
}
#endif
}
#ifdef VIMAGE
else {
/*
* Update the interface index in the link layer address
* of the interface.
*/
for (ifa = ifp->if_addr; ifa != NULL;
ifa = CK_STAILQ_NEXT(ifa, ifa_link)) {
if (ifa->ifa_addr->sa_family == AF_LINK) {
sdl = (struct sockaddr_dl *)ifa->ifa_addr;
sdl->sdl_index = ifp->if_index;
}
}
}
#endif
if_link_ifnet(ifp);
if (domain_init_status >= 2)
if_attachdomain1(ifp);
EVENTHANDLER_INVOKE(ifnet_arrival_event, ifp);
if (IS_DEFAULT_VNET(curvnet))
devctl_notify("IFNET", ifp->if_xname, "ATTACH", NULL);
/* Announce the interface. */
rt_ifannouncemsg(ifp, IFAN_ARRIVAL);
}
static void
if_epochalloc(void *dummy __unused)
{
net_epoch_preempt = epoch_alloc("Net preemptible", EPOCH_PREEMPT);
}
SYSINIT(ifepochalloc, SI_SUB_EPOCH, SI_ORDER_ANY, if_epochalloc, NULL);
static void
if_attachdomain(void *dummy)
{
struct ifnet *ifp;
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link)
if_attachdomain1(ifp);
}
SYSINIT(domainifattach, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_SECOND,
if_attachdomain, NULL);
static void
if_attachdomain1(struct ifnet *ifp)
{
struct domain *dp;
/*
* Since dp->dom_ifattach calls malloc() with M_WAITOK, we
* cannot lock ifp->if_afdata initialization, entirely.
*/
IF_AFDATA_LOCK(ifp);
if (ifp->if_afdata_initialized >= domain_init_status) {
IF_AFDATA_UNLOCK(ifp);
log(LOG_WARNING, "%s called more than once on %s\n",
__func__, ifp->if_xname);
return;
}
ifp->if_afdata_initialized = domain_init_status;
IF_AFDATA_UNLOCK(ifp);
/* address family dependent data region */
bzero(ifp->if_afdata, sizeof(ifp->if_afdata));
for (dp = domains; dp; dp = dp->dom_next) {
if (dp->dom_ifattach)
ifp->if_afdata[dp->dom_family] =
(*dp->dom_ifattach)(ifp);
}
}
/*
* Remove any unicast or broadcast network addresses from an interface.
*/
void
if_purgeaddrs(struct ifnet *ifp)
{
struct ifaddr *ifa;
while (1) {
struct epoch_tracker et;
NET_EPOCH_ENTER(et);
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family != AF_LINK)
break;
}
NET_EPOCH_EXIT(et);
if (ifa == NULL)
break;
#ifdef INET
/* XXX: Ugly!! ad hoc just for INET */
if (ifa->ifa_addr->sa_family == AF_INET) {
struct ifaliasreq ifr;
bzero(&ifr, sizeof(ifr));
ifr.ifra_addr = *ifa->ifa_addr;
if (ifa->ifa_dstaddr)
ifr.ifra_broadaddr = *ifa->ifa_dstaddr;
if (in_control(NULL, SIOCDIFADDR, (caddr_t)&ifr, ifp,
NULL) == 0)
continue;
}
#endif /* INET */
#ifdef INET6
if (ifa->ifa_addr->sa_family == AF_INET6) {
in6_purgeifaddr((struct in6_ifaddr *)ifa);
/* ifp_addrhead is already updated */
continue;
}
#endif /* INET6 */
IF_ADDR_WLOCK(ifp);
CK_STAILQ_REMOVE(&ifp->if_addrhead, ifa, ifaddr, ifa_link);
IF_ADDR_WUNLOCK(ifp);
ifa_free(ifa);
}
}
/*
* Remove any multicast network addresses from an interface when an ifnet
* is going away.
*/
static void
if_purgemaddrs(struct ifnet *ifp)
{
struct ifmultiaddr *ifma;
IF_ADDR_WLOCK(ifp);
while (!CK_STAILQ_EMPTY(&ifp->if_multiaddrs)) {
ifma = CK_STAILQ_FIRST(&ifp->if_multiaddrs);
CK_STAILQ_REMOVE(&ifp->if_multiaddrs, ifma, ifmultiaddr, ifma_link);
if_delmulti_locked(ifp, ifma, 1);
}
IF_ADDR_WUNLOCK(ifp);
}
/*
* Detach an interface, removing it from the list of "active" interfaces.
* If vmove flag is set on entry to if_detach_internal(), perform only a
* limited subset of cleanup tasks, given that we are moving an ifnet from
* one vnet to another, where it must be fully operational.
*
* XXXRW: There are some significant questions about event ordering, and
* how to prevent things from starting to use the interface during detach.
*/
void
if_detach(struct ifnet *ifp)
{
bool found;
CURVNET_SET_QUIET(ifp->if_vnet);
found = if_unlink_ifnet(ifp, false);
if (found) {
sx_xlock(&ifnet_detach_sxlock);
if_detach_internal(ifp, false);
sx_xunlock(&ifnet_detach_sxlock);
}
CURVNET_RESTORE();
}
/*
* The vmove flag, if set, indicates that we are called from a callpath
* that is moving an interface to a different vnet instance.
*
* The shutdown flag, if set, indicates that we are called in the
* process of shutting down a vnet instance. Currently only the
* vnet_if_return SYSUNINIT function sets it. Note: we can be called
* on a vnet instance shutdown without this flag being set, e.g., when
* the cloned interfaces are destoyed as first thing of teardown.
*/
static int
if_detach_internal(struct ifnet *ifp, bool vmove)
{
struct ifaddr *ifa;
int i;
struct domain *dp;
#ifdef VIMAGE
bool shutdown;
shutdown = VNET_IS_SHUTTING_DOWN(ifp->if_vnet);
#endif
/*
* At this point we know the interface still was on the ifnet list
* and we removed it so we are in a stable state.
*/
epoch_wait_preempt(net_epoch_preempt);
/*
* Ensure all pending EPOCH(9) callbacks have been executed. This
* fixes issues about late destruction of multicast options
* which lead to leave group calls, which in turn access the
* belonging ifnet structure:
*/
epoch_drain_callbacks(net_epoch_preempt);
/*
* In any case (destroy or vmove) detach us from the groups
* and remove/wait for pending events on the taskq.
* XXX-BZ in theory an interface could still enqueue a taskq change?
*/
if_delgroups(ifp);
taskqueue_drain(taskqueue_swi, &ifp->if_linktask);
taskqueue_drain(taskqueue_swi, &ifp->if_addmultitask);
if_down(ifp);
#ifdef VIMAGE
/*
* On VNET shutdown abort here as the stack teardown will do all
* the work top-down for us.
*/
if (shutdown) {
/* Give interface users the chance to clean up. */
EVENTHANDLER_INVOKE(ifnet_departure_event, ifp);
/*
* In case of a vmove we are done here without error.
* If we would signal an error it would lead to the same
* abort as if we did not find the ifnet anymore.
* if_detach() calls us in void context and does not care
* about an early abort notification, so life is splendid :)
*/
goto finish_vnet_shutdown;
}
#endif
/*
* At this point we are not tearing down a VNET and are either
* going to destroy or vmove the interface and have to cleanup
* accordingly.
*/
/*
* Remove routes and flush queues.
*/
#ifdef ALTQ
if (ALTQ_IS_ENABLED(&ifp->if_snd))
altq_disable(&ifp->if_snd);
if (ALTQ_IS_ATTACHED(&ifp->if_snd))
altq_detach(&ifp->if_snd);
#endif
if_purgeaddrs(ifp);
#ifdef INET
in_ifdetach(ifp);
#endif
#ifdef INET6
/*
* Remove all IPv6 kernel structs related to ifp. This should be done
* before removing routing entries below, since IPv6 interface direct
* routes are expected to be removed by the IPv6-specific kernel API.
* Otherwise, the kernel will detect some inconsistency and bark it.
*/
in6_ifdetach(ifp);
#endif
if_purgemaddrs(ifp);
/* Announce that the interface is gone. */
rt_ifannouncemsg(ifp, IFAN_DEPARTURE);
EVENTHANDLER_INVOKE(ifnet_departure_event, ifp);
if (IS_DEFAULT_VNET(curvnet))
devctl_notify("IFNET", ifp->if_xname, "DETACH", NULL);
if (!vmove) {
/*
* Prevent further calls into the device driver via ifnet.
*/
if_dead(ifp);
/*
* Clean up all addresses.
*/
IF_ADDR_WLOCK(ifp);
if (!CK_STAILQ_EMPTY(&ifp->if_addrhead)) {
ifa = CK_STAILQ_FIRST(&ifp->if_addrhead);
CK_STAILQ_REMOVE(&ifp->if_addrhead, ifa, ifaddr, ifa_link);
IF_ADDR_WUNLOCK(ifp);
ifa_free(ifa);
} else
IF_ADDR_WUNLOCK(ifp);
}
rt_flushifroutes(ifp);
#ifdef VIMAGE
finish_vnet_shutdown:
#endif
/*
* We cannot hold the lock over dom_ifdetach calls as they might
* sleep, for example trying to drain a callout, thus open up the
* theoretical race with re-attaching.
*/
IF_AFDATA_LOCK(ifp);
i = ifp->if_afdata_initialized;
ifp->if_afdata_initialized = 0;
IF_AFDATA_UNLOCK(ifp);
for (dp = domains; i > 0 && dp; dp = dp->dom_next) {
if (dp->dom_ifdetach && ifp->if_afdata[dp->dom_family]) {
(*dp->dom_ifdetach)(ifp,
ifp->if_afdata[dp->dom_family]);
ifp->if_afdata[dp->dom_family] = NULL;
}
}
return (0);
}
#ifdef VIMAGE
/*
* if_vmove() performs a limited version of if_detach() in current
* vnet and if_attach()es the ifnet to the vnet specified as 2nd arg.
*/
static int
if_vmove(struct ifnet *ifp, struct vnet *new_vnet)
{
#ifdef DEV_BPF
u_int bif_dlt, bif_hdrlen;
#endif
int rc;
#ifdef DEV_BPF
/*
* if_detach_internal() will call the eventhandler to notify
* interface departure. That will detach if_bpf. We need to
* safe the dlt and hdrlen so we can re-attach it later.
*/
bpf_get_bp_params(ifp->if_bpf, &bif_dlt, &bif_hdrlen);
#endif
/*
* Detach from current vnet, but preserve LLADDR info, do not
* mark as dead etc. so that the ifnet can be reattached later.
* If we cannot find it, we lost the race to someone else.
*/
rc = if_detach_internal(ifp, true);
if (rc != 0)
return (rc);
/*
* Perform interface-specific reassignment tasks, if provided by
* the driver.
*/
if (ifp->if_reassign != NULL)
ifp->if_reassign(ifp, new_vnet, NULL);
/*
* Switch to the context of the target vnet.
*/
CURVNET_SET_QUIET(new_vnet);
if_attach_internal(ifp, true);
#ifdef DEV_BPF
if (ifp->if_bpf == NULL)
bpfattach(ifp, bif_dlt, bif_hdrlen);
#endif
CURVNET_RESTORE();
return (0);
}
/*
* Move an ifnet to or from another child prison/vnet, specified by the jail id.
*/
static int
if_vmove_loan(struct thread *td, struct ifnet *ifp, char *ifname, int jid)
{
struct prison *pr;
struct ifnet *difp;
int error;
bool found __diagused;
bool shutdown;
/* Try to find the prison within our visibility. */
sx_slock(&allprison_lock);
pr = prison_find_child(td->td_ucred->cr_prison, jid);
sx_sunlock(&allprison_lock);
if (pr == NULL)
return (ENXIO);
prison_hold_locked(pr);
mtx_unlock(&pr->pr_mtx);
/* Do not try to move the iface from and to the same prison. */
if (pr->pr_vnet == ifp->if_vnet) {
prison_free(pr);
return (EEXIST);
}
/* Make sure the named iface does not exists in the dst. prison/vnet. */
/* XXX Lock interfaces to avoid races. */
CURVNET_SET_QUIET(pr->pr_vnet);
difp = ifunit(ifname);
if (difp != NULL) {
CURVNET_RESTORE();
prison_free(pr);
return (EEXIST);
}
/* Make sure the VNET is stable. */
shutdown = VNET_IS_SHUTTING_DOWN(ifp->if_vnet);
if (shutdown) {
CURVNET_RESTORE();
prison_free(pr);
return (EBUSY);
}
CURVNET_RESTORE();
found = if_unlink_ifnet(ifp, true);
MPASS(found);
/* Move the interface into the child jail/vnet. */
error = if_vmove(ifp, pr->pr_vnet);
/* Report the new if_xname back to the userland on success. */
if (error == 0)
sprintf(ifname, "%s", ifp->if_xname);
prison_free(pr);
return (error);
}
static int
if_vmove_reclaim(struct thread *td, char *ifname, int jid)
{
struct prison *pr;
struct vnet *vnet_dst;
struct ifnet *ifp;
int error, found __diagused;
bool shutdown;
/* Try to find the prison within our visibility. */
sx_slock(&allprison_lock);
pr = prison_find_child(td->td_ucred->cr_prison, jid);
sx_sunlock(&allprison_lock);
if (pr == NULL)
return (ENXIO);
prison_hold_locked(pr);
mtx_unlock(&pr->pr_mtx);
/* Make sure the named iface exists in the source prison/vnet. */
CURVNET_SET(pr->pr_vnet);
ifp = ifunit(ifname); /* XXX Lock to avoid races. */
if (ifp == NULL) {
CURVNET_RESTORE();
prison_free(pr);
return (ENXIO);
}
/* Do not try to move the iface from and to the same prison. */
vnet_dst = TD_TO_VNET(td);
if (vnet_dst == ifp->if_vnet) {
CURVNET_RESTORE();
prison_free(pr);
return (EEXIST);
}
/* Make sure the VNET is stable. */
shutdown = VNET_IS_SHUTTING_DOWN(ifp->if_vnet);
if (shutdown) {
CURVNET_RESTORE();
prison_free(pr);
return (EBUSY);
}
/* Get interface back from child jail/vnet. */
found = if_unlink_ifnet(ifp, true);
MPASS(found);
error = if_vmove(ifp, vnet_dst);
CURVNET_RESTORE();
/* Report the new if_xname back to the userland on success. */
if (error == 0)
sprintf(ifname, "%s", ifp->if_xname);
prison_free(pr);
return (error);
}
#endif /* VIMAGE */
/*
* Add a group to an interface
*/
int
if_addgroup(struct ifnet *ifp, const char *groupname)
{
struct ifg_list *ifgl;
struct ifg_group *ifg = NULL;
struct ifg_member *ifgm;
int new = 0;
if (groupname[0] && groupname[strlen(groupname) - 1] >= '0' &&
groupname[strlen(groupname) - 1] <= '9')
return (EINVAL);
IFNET_WLOCK();
CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next)
if (!strcmp(ifgl->ifgl_group->ifg_group, groupname)) {
IFNET_WUNLOCK();
return (EEXIST);
}
if ((ifgl = malloc(sizeof(*ifgl), M_TEMP, M_NOWAIT)) == NULL) {
IFNET_WUNLOCK();
return (ENOMEM);
}
if ((ifgm = malloc(sizeof(*ifgm), M_TEMP, M_NOWAIT)) == NULL) {
free(ifgl, M_TEMP);
IFNET_WUNLOCK();
return (ENOMEM);
}
CK_STAILQ_FOREACH(ifg, &V_ifg_head, ifg_next)
if (!strcmp(ifg->ifg_group, groupname))
break;
if (ifg == NULL) {
if ((ifg = malloc(sizeof(*ifg), M_TEMP, M_NOWAIT)) == NULL) {
free(ifgl, M_TEMP);
free(ifgm, M_TEMP);
IFNET_WUNLOCK();
return (ENOMEM);
}
strlcpy(ifg->ifg_group, groupname, sizeof(ifg->ifg_group));
ifg->ifg_refcnt = 0;
CK_STAILQ_INIT(&ifg->ifg_members);
CK_STAILQ_INSERT_TAIL(&V_ifg_head, ifg, ifg_next);
new = 1;
}
ifg->ifg_refcnt++;
ifgl->ifgl_group = ifg;
ifgm->ifgm_ifp = ifp;
IF_ADDR_WLOCK(ifp);
CK_STAILQ_INSERT_TAIL(&ifg->ifg_members, ifgm, ifgm_next);
CK_STAILQ_INSERT_TAIL(&ifp->if_groups, ifgl, ifgl_next);
IF_ADDR_WUNLOCK(ifp);
IFNET_WUNLOCK();
if (new)
EVENTHANDLER_INVOKE(group_attach_event, ifg);
EVENTHANDLER_INVOKE(group_change_event, groupname);
return (0);
}
/*
* Helper function to remove a group out of an interface. Expects the global
* ifnet lock to be write-locked, and drops it before returning.
*/
static void
_if_delgroup_locked(struct ifnet *ifp, struct ifg_list *ifgl,
const char *groupname)
{
struct ifg_member *ifgm;
bool freeifgl;
IFNET_WLOCK_ASSERT();
IF_ADDR_WLOCK(ifp);
CK_STAILQ_REMOVE(&ifp->if_groups, ifgl, ifg_list, ifgl_next);
IF_ADDR_WUNLOCK(ifp);
CK_STAILQ_FOREACH(ifgm, &ifgl->ifgl_group->ifg_members, ifgm_next) {
if (ifgm->ifgm_ifp == ifp) {
CK_STAILQ_REMOVE(&ifgl->ifgl_group->ifg_members, ifgm,
ifg_member, ifgm_next);
break;
}
}
if (--ifgl->ifgl_group->ifg_refcnt == 0) {
CK_STAILQ_REMOVE(&V_ifg_head, ifgl->ifgl_group, ifg_group,
ifg_next);
freeifgl = true;
} else {
freeifgl = false;
}
IFNET_WUNLOCK();
epoch_wait_preempt(net_epoch_preempt);
if (freeifgl) {
EVENTHANDLER_INVOKE(group_detach_event, ifgl->ifgl_group);
free(ifgl->ifgl_group, M_TEMP);
}
free(ifgm, M_TEMP);
free(ifgl, M_TEMP);
EVENTHANDLER_INVOKE(group_change_event, groupname);
}
/*
* Remove a group from an interface
*/
int
if_delgroup(struct ifnet *ifp, const char *groupname)
{
struct ifg_list *ifgl;
IFNET_WLOCK();
CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next)
if (strcmp(ifgl->ifgl_group->ifg_group, groupname) == 0)
break;
if (ifgl == NULL) {
IFNET_WUNLOCK();
return (ENOENT);
}
_if_delgroup_locked(ifp, ifgl, groupname);
return (0);
}
/*
* Remove an interface from all groups
*/
static void
if_delgroups(struct ifnet *ifp)
{
struct ifg_list *ifgl;
char groupname[IFNAMSIZ];
IFNET_WLOCK();
while ((ifgl = CK_STAILQ_FIRST(&ifp->if_groups)) != NULL) {
strlcpy(groupname, ifgl->ifgl_group->ifg_group, IFNAMSIZ);
_if_delgroup_locked(ifp, ifgl, groupname);
IFNET_WLOCK();
}
IFNET_WUNLOCK();
}
/*
* Stores all groups from an interface in memory pointed to by ifgr.
*/
static int
if_getgroup(struct ifgroupreq *ifgr, struct ifnet *ifp)
{
int len, error;
struct ifg_list *ifgl;
struct ifg_req ifgrq, *ifgp;
NET_EPOCH_ASSERT();
if (ifgr->ifgr_len == 0) {
CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next)
ifgr->ifgr_len += sizeof(struct ifg_req);
return (0);
}
len = ifgr->ifgr_len;
ifgp = ifgr->ifgr_groups;
/* XXX: wire */
CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) {
if (len < sizeof(ifgrq))
return (EINVAL);
bzero(&ifgrq, sizeof ifgrq);
strlcpy(ifgrq.ifgrq_group, ifgl->ifgl_group->ifg_group,
sizeof(ifgrq.ifgrq_group));
if ((error = copyout(&ifgrq, ifgp, sizeof(struct ifg_req))))
return (error);
len -= sizeof(ifgrq);
ifgp++;
}
return (0);
}
/*
* Stores all members of a group in memory pointed to by igfr
*/
static int
if_getgroupmembers(struct ifgroupreq *ifgr)
{
struct ifg_group *ifg;
struct ifg_member *ifgm;
struct ifg_req ifgrq, *ifgp;
int len, error;
IFNET_RLOCK();
CK_STAILQ_FOREACH(ifg, &V_ifg_head, ifg_next)
if (strcmp(ifg->ifg_group, ifgr->ifgr_name) == 0)
break;
if (ifg == NULL) {
IFNET_RUNLOCK();
return (ENOENT);
}
if (ifgr->ifgr_len == 0) {
CK_STAILQ_FOREACH(ifgm, &ifg->ifg_members, ifgm_next)
ifgr->ifgr_len += sizeof(ifgrq);
IFNET_RUNLOCK();
return (0);
}
len = ifgr->ifgr_len;
ifgp = ifgr->ifgr_groups;
CK_STAILQ_FOREACH(ifgm, &ifg->ifg_members, ifgm_next) {
if (len < sizeof(ifgrq)) {
IFNET_RUNLOCK();
return (EINVAL);
}
bzero(&ifgrq, sizeof ifgrq);
strlcpy(ifgrq.ifgrq_member, ifgm->ifgm_ifp->if_xname,
sizeof(ifgrq.ifgrq_member));
if ((error = copyout(&ifgrq, ifgp, sizeof(struct ifg_req)))) {
IFNET_RUNLOCK();
return (error);
}
len -= sizeof(ifgrq);
ifgp++;
}
IFNET_RUNLOCK();
return (0);
}
/*
* Return counter values from counter(9)s stored in ifnet.
*/
uint64_t
if_get_counter_default(struct ifnet *ifp, ift_counter cnt)
{
KASSERT(cnt < IFCOUNTERS, ("%s: invalid cnt %d", __func__, cnt));
return (counter_u64_fetch(ifp->if_counters[cnt]));
}
/*
* Increase an ifnet counter. Usually used for counters shared
* between the stack and a driver, but function supports them all.
*/
void
if_inc_counter(struct ifnet *ifp, ift_counter cnt, int64_t inc)
{
KASSERT(cnt < IFCOUNTERS, ("%s: invalid cnt %d", __func__, cnt));
counter_u64_add(ifp->if_counters[cnt], inc);
}
/*
* Copy data from ifnet to userland API structure if_data.
*/
void
if_data_copy(struct ifnet *ifp, struct if_data *ifd)
{
ifd->ifi_type = ifp->if_type;
ifd->ifi_physical = 0;
ifd->ifi_addrlen = ifp->if_addrlen;
ifd->ifi_hdrlen = ifp->if_hdrlen;
ifd->ifi_link_state = ifp->if_link_state;
ifd->ifi_vhid = 0;
ifd->ifi_datalen = sizeof(struct if_data);
ifd->ifi_mtu = ifp->if_mtu;
ifd->ifi_metric = ifp->if_metric;
ifd->ifi_baudrate = ifp->if_baudrate;
ifd->ifi_hwassist = ifp->if_hwassist;
ifd->ifi_epoch = ifp->if_epoch;
ifd->ifi_lastchange = ifp->if_lastchange;
ifd->ifi_ipackets = ifp->if_get_counter(ifp, IFCOUNTER_IPACKETS);
ifd->ifi_ierrors = ifp->if_get_counter(ifp, IFCOUNTER_IERRORS);
ifd->ifi_opackets = ifp->if_get_counter(ifp, IFCOUNTER_OPACKETS);
ifd->ifi_oerrors = ifp->if_get_counter(ifp, IFCOUNTER_OERRORS);
ifd->ifi_collisions = ifp->if_get_counter(ifp, IFCOUNTER_COLLISIONS);
ifd->ifi_ibytes = ifp->if_get_counter(ifp, IFCOUNTER_IBYTES);
ifd->ifi_obytes = ifp->if_get_counter(ifp, IFCOUNTER_OBYTES);
ifd->ifi_imcasts = ifp->if_get_counter(ifp, IFCOUNTER_IMCASTS);
ifd->ifi_omcasts = ifp->if_get_counter(ifp, IFCOUNTER_OMCASTS);
ifd->ifi_iqdrops = ifp->if_get_counter(ifp, IFCOUNTER_IQDROPS);
ifd->ifi_oqdrops = ifp->if_get_counter(ifp, IFCOUNTER_OQDROPS);
ifd->ifi_noproto = ifp->if_get_counter(ifp, IFCOUNTER_NOPROTO);
}
/*
* Initialization, destruction and refcounting functions for ifaddrs.
*/
struct ifaddr *
ifa_alloc(size_t size, int flags)
{
struct ifaddr *ifa;
KASSERT(size >= sizeof(struct ifaddr),
("%s: invalid size %zu", __func__, size));
ifa = malloc(size, M_IFADDR, M_ZERO | flags);
if (ifa == NULL)
return (NULL);
if ((ifa->ifa_opackets = counter_u64_alloc(flags)) == NULL)
goto fail;
if ((ifa->ifa_ipackets = counter_u64_alloc(flags)) == NULL)
goto fail;
if ((ifa->ifa_obytes = counter_u64_alloc(flags)) == NULL)
goto fail;
if ((ifa->ifa_ibytes = counter_u64_alloc(flags)) == NULL)
goto fail;
refcount_init(&ifa->ifa_refcnt, 1);
return (ifa);
fail:
/* free(NULL) is okay */
counter_u64_free(ifa->ifa_opackets);
counter_u64_free(ifa->ifa_ipackets);
counter_u64_free(ifa->ifa_obytes);
counter_u64_free(ifa->ifa_ibytes);
free(ifa, M_IFADDR);
return (NULL);
}
void
ifa_ref(struct ifaddr *ifa)
{
u_int old __diagused;
old = refcount_acquire(&ifa->ifa_refcnt);
KASSERT(old > 0, ("%s: ifa %p has 0 refs", __func__, ifa));
}
int
ifa_try_ref(struct ifaddr *ifa)
{
NET_EPOCH_ASSERT();
return (refcount_acquire_if_not_zero(&ifa->ifa_refcnt));
}
static void
ifa_destroy(epoch_context_t ctx)
{
struct ifaddr *ifa;
ifa = __containerof(ctx, struct ifaddr, ifa_epoch_ctx);
counter_u64_free(ifa->ifa_opackets);
counter_u64_free(ifa->ifa_ipackets);
counter_u64_free(ifa->ifa_obytes);
counter_u64_free(ifa->ifa_ibytes);
free(ifa, M_IFADDR);
}
void
ifa_free(struct ifaddr *ifa)
{
if (refcount_release(&ifa->ifa_refcnt))
NET_EPOCH_CALL(ifa_destroy, &ifa->ifa_epoch_ctx);
}
/*
* XXX: Because sockaddr_dl has deeper structure than the sockaddr
* structs used to represent other address families, it is necessary
* to perform a different comparison.
*/
#define sa_dl_equal(a1, a2) \
((((const struct sockaddr_dl *)(a1))->sdl_len == \
((const struct sockaddr_dl *)(a2))->sdl_len) && \
(bcmp(CLLADDR((const struct sockaddr_dl *)(a1)), \
CLLADDR((const struct sockaddr_dl *)(a2)), \
((const struct sockaddr_dl *)(a1))->sdl_alen) == 0))
/*
* Locate an interface based on a complete address.
*/
/*ARGSUSED*/
struct ifaddr *
ifa_ifwithaddr(const struct sockaddr *addr)
{
struct ifnet *ifp;
struct ifaddr *ifa;
NET_EPOCH_ASSERT();
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family != addr->sa_family)
continue;
if (sa_equal(addr, ifa->ifa_addr)) {
goto done;
}
/* IP6 doesn't have broadcast */
if ((ifp->if_flags & IFF_BROADCAST) &&
ifa->ifa_broadaddr &&
ifa->ifa_broadaddr->sa_len != 0 &&
sa_equal(ifa->ifa_broadaddr, addr)) {
goto done;
}
}
}
ifa = NULL;
done:
return (ifa);
}
int
ifa_ifwithaddr_check(const struct sockaddr *addr)
{
struct epoch_tracker et;
int rc;
NET_EPOCH_ENTER(et);
rc = (ifa_ifwithaddr(addr) != NULL);
NET_EPOCH_EXIT(et);
return (rc);
}
/*
* Locate an interface based on the broadcast address.
*/
/* ARGSUSED */
struct ifaddr *
ifa_ifwithbroadaddr(const struct sockaddr *addr, int fibnum)
{
struct ifnet *ifp;
struct ifaddr *ifa;
NET_EPOCH_ASSERT();
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if ((fibnum != RT_ALL_FIBS) && (ifp->if_fib != fibnum))
continue;
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family != addr->sa_family)
continue;
if ((ifp->if_flags & IFF_BROADCAST) &&
ifa->ifa_broadaddr &&
ifa->ifa_broadaddr->sa_len != 0 &&
sa_equal(ifa->ifa_broadaddr, addr)) {
goto done;
}
}
}
ifa = NULL;
done:
return (ifa);
}
/*
* Locate the point to point interface with a given destination address.
*/
/*ARGSUSED*/
struct ifaddr *
ifa_ifwithdstaddr(const struct sockaddr *addr, int fibnum)
{
struct ifnet *ifp;
struct ifaddr *ifa;
NET_EPOCH_ASSERT();
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if ((ifp->if_flags & IFF_POINTOPOINT) == 0)
continue;
if ((fibnum != RT_ALL_FIBS) && (ifp->if_fib != fibnum))
continue;
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family != addr->sa_family)
continue;
if (ifa->ifa_dstaddr != NULL &&
sa_equal(addr, ifa->ifa_dstaddr)) {
goto done;
}
}
}
ifa = NULL;
done:
return (ifa);
}
/*
* Find an interface on a specific network. If many, choice
* is most specific found.
*/
struct ifaddr *
ifa_ifwithnet(const struct sockaddr *addr, int ignore_ptp, int fibnum)
{
struct ifnet *ifp;
struct ifaddr *ifa;
struct ifaddr *ifa_maybe = NULL;
u_int af = addr->sa_family;
const char *addr_data = addr->sa_data, *cplim;
NET_EPOCH_ASSERT();
/*
* AF_LINK addresses can be looked up directly by their index number,
* so do that if we can.
*/
if (af == AF_LINK) {
ifp = ifnet_byindex(
((const struct sockaddr_dl *)addr)->sdl_index);
return (ifp ? ifp->if_addr : NULL);
}
/*
* Scan though each interface, looking for ones that have addresses
* in this address family and the requested fib.
*/
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if ((fibnum != RT_ALL_FIBS) && (ifp->if_fib != fibnum))
continue;
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
const char *cp, *cp2, *cp3;
if (ifa->ifa_addr->sa_family != af)
next: continue;
if (af == AF_INET &&
ifp->if_flags & IFF_POINTOPOINT && !ignore_ptp) {
/*
* This is a bit broken as it doesn't
* take into account that the remote end may
* be a single node in the network we are
* looking for.
* The trouble is that we don't know the
* netmask for the remote end.
*/
if (ifa->ifa_dstaddr != NULL &&
sa_equal(addr, ifa->ifa_dstaddr)) {
goto done;
}
} else {
/*
* Scan all the bits in the ifa's address.
* If a bit dissagrees with what we are
* looking for, mask it with the netmask
* to see if it really matters.
* (A byte at a time)
*/
if (ifa->ifa_netmask == 0)
continue;
cp = addr_data;
cp2 = ifa->ifa_addr->sa_data;
cp3 = ifa->ifa_netmask->sa_data;
cplim = ifa->ifa_netmask->sa_len
+ (char *)ifa->ifa_netmask;
while (cp3 < cplim)
if ((*cp++ ^ *cp2++) & *cp3++)
goto next; /* next address! */
/*
* If the netmask of what we just found
* is more specific than what we had before
* (if we had one), or if the virtual status
* of new prefix is better than of the old one,
* then remember the new one before continuing
* to search for an even better one.
*/
if (ifa_maybe == NULL ||
ifa_preferred(ifa_maybe, ifa) ||
rn_refines((caddr_t)ifa->ifa_netmask,
(caddr_t)ifa_maybe->ifa_netmask)) {
ifa_maybe = ifa;
}
}
}
}
ifa = ifa_maybe;
ifa_maybe = NULL;
done:
return (ifa);
}
/*
* Find an interface address specific to an interface best matching
* a given address.
*/
struct ifaddr *
ifaof_ifpforaddr(const struct sockaddr *addr, struct ifnet *ifp)
{
struct ifaddr *ifa;
const char *cp, *cp2, *cp3;
char *cplim;
struct ifaddr *ifa_maybe = NULL;
u_int af = addr->sa_family;
if (af >= AF_MAX)
return (NULL);
NET_EPOCH_ASSERT();
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family != af)
continue;
if (ifa_maybe == NULL)
ifa_maybe = ifa;
if (ifa->ifa_netmask == 0) {
if (sa_equal(addr, ifa->ifa_addr) ||
(ifa->ifa_dstaddr &&
sa_equal(addr, ifa->ifa_dstaddr)))
goto done;
continue;
}
if (ifp->if_flags & IFF_POINTOPOINT) {
if (sa_equal(addr, ifa->ifa_dstaddr))
goto done;
} else {
cp = addr->sa_data;
cp2 = ifa->ifa_addr->sa_data;
cp3 = ifa->ifa_netmask->sa_data;
cplim = ifa->ifa_netmask->sa_len + (char *)ifa->ifa_netmask;
for (; cp3 < cplim; cp3++)
if ((*cp++ ^ *cp2++) & *cp3)
break;
if (cp3 == cplim)
goto done;
}
}
ifa = ifa_maybe;
done:
return (ifa);
}
/*
* See whether new ifa is better than current one:
* 1) A non-virtual one is preferred over virtual.
* 2) A virtual in master state preferred over any other state.
*
* Used in several address selecting functions.
*/
int
ifa_preferred(struct ifaddr *cur, struct ifaddr *next)
{
return (cur->ifa_carp && (!next->ifa_carp ||
((*carp_master_p)(next) && !(*carp_master_p)(cur))));
}
struct sockaddr_dl *
link_alloc_sdl(size_t size, int flags)
{
return (malloc(size, M_TEMP, flags));
}
void
link_free_sdl(struct sockaddr *sa)
{
free(sa, M_TEMP);
}
/*
* Fills in given sdl with interface basic info.
* Returns pointer to filled sdl.
*/
struct sockaddr_dl *
link_init_sdl(struct ifnet *ifp, struct sockaddr *paddr, u_char iftype)
{
struct sockaddr_dl *sdl;
sdl = (struct sockaddr_dl *)paddr;
memset(sdl, 0, sizeof(struct sockaddr_dl));
sdl->sdl_len = sizeof(struct sockaddr_dl);
sdl->sdl_family = AF_LINK;
sdl->sdl_index = ifp->if_index;
sdl->sdl_type = iftype;
return (sdl);
}
/*
* Mark an interface down and notify protocols of
* the transition.
*/
static void
if_unroute(struct ifnet *ifp, int flag, int fam)
{
struct ifaddr *ifa;
struct epoch_tracker et;
KASSERT(flag == IFF_UP, ("if_unroute: flag != IFF_UP"));
ifp->if_flags &= ~flag;
getmicrotime(&ifp->if_lastchange);
NET_EPOCH_ENTER(et);
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link)
if (fam == PF_UNSPEC || (fam == ifa->ifa_addr->sa_family))
pfctlinput(PRC_IFDOWN, ifa->ifa_addr);
NET_EPOCH_EXIT(et);
ifp->if_qflush(ifp);
if (ifp->if_carp)
(*carp_linkstate_p)(ifp);
rt_ifmsg(ifp);
}
/*
* Mark an interface up and notify protocols of
* the transition.
*/
static void
if_route(struct ifnet *ifp, int flag, int fam)
{
struct ifaddr *ifa;
struct epoch_tracker et;
KASSERT(flag == IFF_UP, ("if_route: flag != IFF_UP"));
ifp->if_flags |= flag;
getmicrotime(&ifp->if_lastchange);
NET_EPOCH_ENTER(et);
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link)
if (fam == PF_UNSPEC || (fam == ifa->ifa_addr->sa_family))
pfctlinput(PRC_IFUP, ifa->ifa_addr);
NET_EPOCH_EXIT(et);
if (ifp->if_carp)
(*carp_linkstate_p)(ifp);
rt_ifmsg(ifp);
#ifdef INET6
in6_if_up(ifp);
#endif
}
void (*vlan_link_state_p)(struct ifnet *); /* XXX: private from if_vlan */
void (*vlan_trunk_cap_p)(struct ifnet *); /* XXX: private from if_vlan */
struct ifnet *(*vlan_trunkdev_p)(struct ifnet *);
struct ifnet *(*vlan_devat_p)(struct ifnet *, uint16_t);
int (*vlan_tag_p)(struct ifnet *, uint16_t *);
int (*vlan_pcp_p)(struct ifnet *, uint16_t *);
int (*vlan_setcookie_p)(struct ifnet *, void *);
void *(*vlan_cookie_p)(struct ifnet *);
/*
* Handle a change in the interface link state. To avoid LORs
* between driver lock and upper layer locks, as well as possible
* recursions, we post event to taskqueue, and all job
* is done in static do_link_state_change().
*/
void
if_link_state_change(struct ifnet *ifp, int link_state)
{
/* Return if state hasn't changed. */
if (ifp->if_link_state == link_state)
return;
ifp->if_link_state = link_state;
/* XXXGL: reference ifp? */
taskqueue_enqueue(taskqueue_swi, &ifp->if_linktask);
}
static void
do_link_state_change(void *arg, int pending)
{
struct ifnet *ifp;
int link_state;
ifp = arg;
link_state = ifp->if_link_state;
CURVNET_SET(ifp->if_vnet);
rt_ifmsg(ifp);
if (ifp->if_vlantrunk != NULL)
(*vlan_link_state_p)(ifp);
if ((ifp->if_type == IFT_ETHER || ifp->if_type == IFT_L2VLAN) &&
ifp->if_l2com != NULL)
(*ng_ether_link_state_p)(ifp, link_state);
if (ifp->if_carp)
(*carp_linkstate_p)(ifp);
if (ifp->if_bridge)
ifp->if_bridge_linkstate(ifp);
if (ifp->if_lagg)
(*lagg_linkstate_p)(ifp, link_state);
if (IS_DEFAULT_VNET(curvnet))
devctl_notify("IFNET", ifp->if_xname,
(link_state == LINK_STATE_UP) ? "LINK_UP" : "LINK_DOWN",
NULL);
if (pending > 1)
if_printf(ifp, "%d link states coalesced\n", pending);
if (log_link_state_change)
if_printf(ifp, "link state changed to %s\n",
(link_state == LINK_STATE_UP) ? "UP" : "DOWN" );
EVENTHANDLER_INVOKE(ifnet_link_event, ifp, link_state);
CURVNET_RESTORE();
}
/*
* Mark an interface down and notify protocols of
* the transition.
*/
void
if_down(struct ifnet *ifp)
{
EVENTHANDLER_INVOKE(ifnet_event, ifp, IFNET_EVENT_DOWN);
if_unroute(ifp, IFF_UP, AF_UNSPEC);
}
/*
* Mark an interface up and notify protocols of
* the transition.
*/
void
if_up(struct ifnet *ifp)
{
if_route(ifp, IFF_UP, AF_UNSPEC);
EVENTHANDLER_INVOKE(ifnet_event, ifp, IFNET_EVENT_UP);
}
/*
* Flush an interface queue.
*/
void
if_qflush(struct ifnet *ifp)
{
struct mbuf *m, *n;
struct ifaltq *ifq;
ifq = &ifp->if_snd;
IFQ_LOCK(ifq);
#ifdef ALTQ
if (ALTQ_IS_ENABLED(ifq))
ALTQ_PURGE(ifq);
#endif
n = ifq->ifq_head;
while ((m = n) != NULL) {
n = m->m_nextpkt;
m_freem(m);
}
ifq->ifq_head = 0;
ifq->ifq_tail = 0;
ifq->ifq_len = 0;
IFQ_UNLOCK(ifq);
}
/*
* Map interface name to interface structure pointer, with or without
* returning a reference.
*/
struct ifnet *
ifunit_ref(const char *name)
{
struct epoch_tracker et;
struct ifnet *ifp;
NET_EPOCH_ENTER(et);
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if (strncmp(name, ifp->if_xname, IFNAMSIZ) == 0 &&
!(ifp->if_flags & IFF_DYING))
break;
}
if (ifp != NULL)
if_ref(ifp);
NET_EPOCH_EXIT(et);
return (ifp);
}
struct ifnet *
ifunit(const char *name)
{
struct epoch_tracker et;
struct ifnet *ifp;
NET_EPOCH_ENTER(et);
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if (strncmp(name, ifp->if_xname, IFNAMSIZ) == 0)
break;
}
NET_EPOCH_EXIT(et);
return (ifp);
}
void *
ifr_buffer_get_buffer(void *data)
{
union ifreq_union *ifrup;
ifrup = data;
#ifdef COMPAT_FREEBSD32
if (SV_CURPROC_FLAG(SV_ILP32))
return ((void *)(uintptr_t)
ifrup->ifr32.ifr_ifru.ifru_buffer.buffer);
#endif
return (ifrup->ifr.ifr_ifru.ifru_buffer.buffer);
}
static void
ifr_buffer_set_buffer_null(void *data)
{
union ifreq_union *ifrup;
ifrup = data;
#ifdef COMPAT_FREEBSD32
if (SV_CURPROC_FLAG(SV_ILP32))
ifrup->ifr32.ifr_ifru.ifru_buffer.buffer = 0;
else
#endif
ifrup->ifr.ifr_ifru.ifru_buffer.buffer = NULL;
}
size_t
ifr_buffer_get_length(void *data)
{
union ifreq_union *ifrup;
ifrup = data;
#ifdef COMPAT_FREEBSD32
if (SV_CURPROC_FLAG(SV_ILP32))
return (ifrup->ifr32.ifr_ifru.ifru_buffer.length);
#endif
return (ifrup->ifr.ifr_ifru.ifru_buffer.length);
}
static void
ifr_buffer_set_length(void *data, size_t len)
{
union ifreq_union *ifrup;
ifrup = data;
#ifdef COMPAT_FREEBSD32
if (SV_CURPROC_FLAG(SV_ILP32))
ifrup->ifr32.ifr_ifru.ifru_buffer.length = len;
else
#endif
ifrup->ifr.ifr_ifru.ifru_buffer.length = len;
}
void *
ifr_data_get_ptr(void *ifrp)
{
union ifreq_union *ifrup;
ifrup = ifrp;
#ifdef COMPAT_FREEBSD32
if (SV_CURPROC_FLAG(SV_ILP32))
return ((void *)(uintptr_t)
ifrup->ifr32.ifr_ifru.ifru_data);
#endif
return (ifrup->ifr.ifr_ifru.ifru_data);
}
/*
* Hardware specific interface ioctls.
*/
int
ifhwioctl(u_long cmd, struct ifnet *ifp, caddr_t data, struct thread *td)
{
struct ifreq *ifr;
int error = 0, do_ifup = 0;
int new_flags, temp_flags;
size_t namelen, onamelen;
size_t descrlen;
char *descrbuf, *odescrbuf;
char new_name[IFNAMSIZ];
char old_name[IFNAMSIZ], strbuf[IFNAMSIZ + 8];
struct ifaddr *ifa;
struct sockaddr_dl *sdl;
ifr = (struct ifreq *)data;
switch (cmd) {
case SIOCGIFINDEX:
ifr->ifr_index = ifp->if_index;
break;
case SIOCGIFFLAGS:
temp_flags = ifp->if_flags | ifp->if_drv_flags;
ifr->ifr_flags = temp_flags & 0xffff;
ifr->ifr_flagshigh = temp_flags >> 16;
break;
case SIOCGIFCAP:
ifr->ifr_reqcap = ifp->if_capabilities;
ifr->ifr_curcap = ifp->if_capenable;
break;
case SIOCGIFDATA:
{
struct if_data ifd;
/* Ensure uninitialised padding is not leaked. */
memset(&ifd, 0, sizeof(ifd));
if_data_copy(ifp, &ifd);
error = copyout(&ifd, ifr_data_get_ptr(ifr), sizeof(ifd));
break;
}
#ifdef MAC
case SIOCGIFMAC:
error = mac_ifnet_ioctl_get(td->td_ucred, ifr, ifp);
break;
#endif
case SIOCGIFMETRIC:
ifr->ifr_metric = ifp->if_metric;
break;
case SIOCGIFMTU:
ifr->ifr_mtu = ifp->if_mtu;
break;
case SIOCGIFPHYS:
/* XXXGL: did this ever worked? */
ifr->ifr_phys = 0;
break;
case SIOCGIFDESCR:
error = 0;
sx_slock(&ifdescr_sx);
if (ifp->if_description == NULL)
error = ENOMSG;
else {
/* space for terminating nul */
descrlen = strlen(ifp->if_description) + 1;
if (ifr_buffer_get_length(ifr) < descrlen)
ifr_buffer_set_buffer_null(ifr);
else
error = copyout(ifp->if_description,
ifr_buffer_get_buffer(ifr), descrlen);
ifr_buffer_set_length(ifr, descrlen);
}
sx_sunlock(&ifdescr_sx);
break;
case SIOCSIFDESCR:
error = priv_check(td, PRIV_NET_SETIFDESCR);
if (error)
return (error);
/*
* Copy only (length-1) bytes to make sure that
* if_description is always nul terminated. The
* length parameter is supposed to count the
* terminating nul in.
*/
if (ifr_buffer_get_length(ifr) > ifdescr_maxlen)
return (ENAMETOOLONG);
else if (ifr_buffer_get_length(ifr) == 0)
descrbuf = NULL;
else {
descrbuf = malloc(ifr_buffer_get_length(ifr),
M_IFDESCR, M_WAITOK | M_ZERO);
error = copyin(ifr_buffer_get_buffer(ifr), descrbuf,
ifr_buffer_get_length(ifr) - 1);
if (error) {
free(descrbuf, M_IFDESCR);
break;
}
}
sx_xlock(&ifdescr_sx);
odescrbuf = ifp->if_description;
ifp->if_description = descrbuf;
sx_xunlock(&ifdescr_sx);
getmicrotime(&ifp->if_lastchange);
free(odescrbuf, M_IFDESCR);
break;
case SIOCGIFFIB:
ifr->ifr_fib = ifp->if_fib;
break;
case SIOCSIFFIB:
error = priv_check(td, PRIV_NET_SETIFFIB);
if (error)
return (error);
if (ifr->ifr_fib >= rt_numfibs)
return (EINVAL);
ifp->if_fib = ifr->ifr_fib;
break;
case SIOCSIFFLAGS:
error = priv_check(td, PRIV_NET_SETIFFLAGS);
if (error)
return (error);
/*
* Currently, no driver owned flags pass the IFF_CANTCHANGE
* check, so we don't need special handling here yet.
*/
new_flags = (ifr->ifr_flags & 0xffff) |
(ifr->ifr_flagshigh << 16);
if (ifp->if_flags & IFF_UP &&
(new_flags & IFF_UP) == 0) {
if_down(ifp);
} else if (new_flags & IFF_UP &&
(ifp->if_flags & IFF_UP) == 0) {
do_ifup = 1;
}
/* See if permanently promiscuous mode bit is about to flip */
if ((ifp->if_flags ^ new_flags) & IFF_PPROMISC) {
if (new_flags & IFF_PPROMISC)
ifp->if_flags |= IFF_PROMISC;
else if (ifp->if_pcount == 0)
ifp->if_flags &= ~IFF_PROMISC;
if (log_promisc_mode_change)
if_printf(ifp, "permanently promiscuous mode %s\n",
((new_flags & IFF_PPROMISC) ?
"enabled" : "disabled"));
}
ifp->if_flags = (ifp->if_flags & IFF_CANTCHANGE) |
(new_flags &~ IFF_CANTCHANGE);
if (ifp->if_ioctl) {
(void) (*ifp->if_ioctl)(ifp, cmd, data);
}
if (do_ifup)
if_up(ifp);
getmicrotime(&ifp->if_lastchange);
break;
case SIOCSIFCAP:
error = priv_check(td, PRIV_NET_SETIFCAP);
if (error)
return (error);
if (ifp->if_ioctl == NULL)
return (EOPNOTSUPP);
if (ifr->ifr_reqcap & ~ifp->if_capabilities)
return (EINVAL);
error = (*ifp->if_ioctl)(ifp, cmd, data);
if (error == 0)
getmicrotime(&ifp->if_lastchange);
break;
#ifdef MAC
case SIOCSIFMAC:
error = mac_ifnet_ioctl_set(td->td_ucred, ifr, ifp);
break;
#endif
case SIOCSIFNAME:
error = priv_check(td, PRIV_NET_SETIFNAME);
if (error)
return (error);
error = copyinstr(ifr_data_get_ptr(ifr), new_name, IFNAMSIZ,
NULL);
if (error != 0)
return (error);
if (new_name[0] == '\0')
return (EINVAL);
if (strcmp(new_name, ifp->if_xname) == 0)
break;
if (ifunit(new_name) != NULL)
return (EEXIST);
/*
* XXX: Locking. Nothing else seems to lock if_flags,
* and there are numerous other races with the
* ifunit() checks not being atomic with namespace
* changes (renames, vmoves, if_attach, etc).
*/
ifp->if_flags |= IFF_RENAMING;
/* Announce the departure of the interface. */
rt_ifannouncemsg(ifp, IFAN_DEPARTURE);
EVENTHANDLER_INVOKE(ifnet_departure_event, ifp);
if_printf(ifp, "changing name to '%s'\n", new_name);
IF_ADDR_WLOCK(ifp);
strlcpy(old_name, ifp->if_xname, sizeof(old_name));
strlcpy(ifp->if_xname, new_name, sizeof(ifp->if_xname));
ifa = ifp->if_addr;
sdl = (struct sockaddr_dl *)ifa->ifa_addr;
namelen = strlen(new_name);
onamelen = sdl->sdl_nlen;
/*
* Move the address if needed. This is safe because we
* allocate space for a name of length IFNAMSIZ when we
* create this in if_attach().
*/
if (namelen != onamelen) {
bcopy(sdl->sdl_data + onamelen,
sdl->sdl_data + namelen, sdl->sdl_alen);
}
bcopy(new_name, sdl->sdl_data, namelen);
sdl->sdl_nlen = namelen;
sdl = (struct sockaddr_dl *)ifa->ifa_netmask;
bzero(sdl->sdl_data, onamelen);
while (namelen != 0)
sdl->sdl_data[--namelen] = 0xff;
IF_ADDR_WUNLOCK(ifp);
EVENTHANDLER_INVOKE(ifnet_arrival_event, ifp);
/* Announce the return of the interface. */
rt_ifannouncemsg(ifp, IFAN_ARRIVAL);
ifp->if_flags &= ~IFF_RENAMING;
snprintf(strbuf, sizeof(strbuf), "name=%s", new_name);
devctl_notify("IFNET", old_name, "RENAME", strbuf);
break;
#ifdef VIMAGE
case SIOCSIFVNET:
error = priv_check(td, PRIV_NET_SETIFVNET);
if (error)
return (error);
error = if_vmove_loan(td, ifp, ifr->ifr_name, ifr->ifr_jid);
break;
#endif
case SIOCSIFMETRIC:
error = priv_check(td, PRIV_NET_SETIFMETRIC);
if (error)
return (error);
ifp->if_metric = ifr->ifr_metric;
getmicrotime(&ifp->if_lastchange);
break;
case SIOCSIFPHYS:
error = priv_check(td, PRIV_NET_SETIFPHYS);
if (error)
return (error);
if (ifp->if_ioctl == NULL)
return (EOPNOTSUPP);
error = (*ifp->if_ioctl)(ifp, cmd, data);
if (error == 0)
getmicrotime(&ifp->if_lastchange);
break;
case SIOCSIFMTU:
{
u_long oldmtu = ifp->if_mtu;
error = priv_check(td, PRIV_NET_SETIFMTU);
if (error)
return (error);
if (ifr->ifr_mtu < IF_MINMTU || ifr->ifr_mtu > IF_MAXMTU)
return (EINVAL);
if (ifp->if_ioctl == NULL)
return (EOPNOTSUPP);
/* Disallow MTU changes on bridge member interfaces. */
if (ifp->if_bridge)
return (EOPNOTSUPP);
error = (*ifp->if_ioctl)(ifp, cmd, data);
if (error == 0) {
getmicrotime(&ifp->if_lastchange);
rt_ifmsg(ifp);
#ifdef INET
DEBUGNET_NOTIFY_MTU(ifp);
#endif
}
/*
* If the link MTU changed, do network layer specific procedure.
*/
if (ifp->if_mtu != oldmtu) {
#ifdef INET6
nd6_setmtu(ifp);
#endif
rt_updatemtu(ifp);
}
break;
}
case SIOCADDMULTI:
case SIOCDELMULTI:
if (cmd == SIOCADDMULTI)
error = priv_check(td, PRIV_NET_ADDMULTI);
else
error = priv_check(td, PRIV_NET_DELMULTI);
if (error)
return (error);
/* Don't allow group membership on non-multicast interfaces. */
if ((ifp->if_flags & IFF_MULTICAST) == 0)
return (EOPNOTSUPP);
/* Don't let users screw up protocols' entries. */
if (ifr->ifr_addr.sa_family != AF_LINK)
return (EINVAL);
if (cmd == SIOCADDMULTI) {
struct epoch_tracker et;
struct ifmultiaddr *ifma;
/*
* Userland is only permitted to join groups once
* via the if_addmulti() KPI, because it cannot hold
* struct ifmultiaddr * between calls. It may also
* lose a race while we check if the membership
* already exists.
*/
NET_EPOCH_ENTER(et);
ifma = if_findmulti(ifp, &ifr->ifr_addr);
NET_EPOCH_EXIT(et);
if (ifma != NULL)
error = EADDRINUSE;
else
error = if_addmulti(ifp, &ifr->ifr_addr, &ifma);
} else {
error = if_delmulti(ifp, &ifr->ifr_addr);
}
if (error == 0)
getmicrotime(&ifp->if_lastchange);
break;
case SIOCSIFPHYADDR:
case SIOCDIFPHYADDR:
#ifdef INET6
case SIOCSIFPHYADDR_IN6:
#endif
case SIOCSIFMEDIA:
case SIOCSIFGENERIC:
error = priv_check(td, PRIV_NET_HWIOCTL);
if (error)
return (error);
if (ifp->if_ioctl == NULL)
return (EOPNOTSUPP);
error = (*ifp->if_ioctl)(ifp, cmd, data);
if (error == 0)
getmicrotime(&ifp->if_lastchange);
break;
case SIOCGIFSTATUS:
case SIOCGIFPSRCADDR:
case SIOCGIFPDSTADDR:
case SIOCGIFMEDIA:
case SIOCGIFXMEDIA:
case SIOCGIFGENERIC:
case SIOCGIFRSSKEY:
case SIOCGIFRSSHASH:
case SIOCGIFDOWNREASON:
if (ifp->if_ioctl == NULL)
return (EOPNOTSUPP);
error = (*ifp->if_ioctl)(ifp, cmd, data);
break;
case SIOCSIFLLADDR:
error = priv_check(td, PRIV_NET_SETLLADDR);
if (error)
return (error);
error = if_setlladdr(ifp,
ifr->ifr_addr.sa_data, ifr->ifr_addr.sa_len);
break;
case SIOCGHWADDR:
error = if_gethwaddr(ifp, ifr);
break;
case SIOCAIFGROUP:
error = priv_check(td, PRIV_NET_ADDIFGROUP);
if (error)
return (error);
error = if_addgroup(ifp,
((struct ifgroupreq *)data)->ifgr_group);
if (error != 0)
return (error);
break;
case SIOCGIFGROUP:
{
struct epoch_tracker et;
NET_EPOCH_ENTER(et);
error = if_getgroup((struct ifgroupreq *)data, ifp);
NET_EPOCH_EXIT(et);
break;
}
case SIOCDIFGROUP:
error = priv_check(td, PRIV_NET_DELIFGROUP);
if (error)
return (error);
error = if_delgroup(ifp,
((struct ifgroupreq *)data)->ifgr_group);
if (error != 0)
return (error);
break;
default:
error = ENOIOCTL;
break;
}
return (error);
}
/*
* Interface ioctls.
*/
int
ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td)
{
#ifdef COMPAT_FREEBSD32
union {
struct ifconf ifc;
struct ifdrv ifd;
struct ifgroupreq ifgr;
struct ifmediareq ifmr;
} thunk;
u_long saved_cmd;
struct ifconf32 *ifc32;
struct ifdrv32 *ifd32;
struct ifgroupreq32 *ifgr32;
struct ifmediareq32 *ifmr32;
#endif
struct ifnet *ifp;
struct ifreq *ifr;
int error;
int oif_flags;
#ifdef VIMAGE
bool shutdown;
#endif
CURVNET_SET(so->so_vnet);
#ifdef VIMAGE
/* Make sure the VNET is stable. */
shutdown = VNET_IS_SHUTTING_DOWN(so->so_vnet);
if (shutdown) {
CURVNET_RESTORE();
return (EBUSY);
}
#endif
#ifdef COMPAT_FREEBSD32
saved_cmd = cmd;
switch (cmd) {
case SIOCGIFCONF32:
ifc32 = (struct ifconf32 *)data;
thunk.ifc.ifc_len = ifc32->ifc_len;
thunk.ifc.ifc_buf = PTRIN(ifc32->ifc_buf);
data = (caddr_t)&thunk.ifc;
cmd = SIOCGIFCONF;
break;
case SIOCGDRVSPEC32:
case SIOCSDRVSPEC32:
ifd32 = (struct ifdrv32 *)data;
memcpy(thunk.ifd.ifd_name, ifd32->ifd_name,
sizeof(thunk.ifd.ifd_name));
thunk.ifd.ifd_cmd = ifd32->ifd_cmd;
thunk.ifd.ifd_len = ifd32->ifd_len;
thunk.ifd.ifd_data = PTRIN(ifd32->ifd_data);
data = (caddr_t)&thunk.ifd;
cmd = _IOC_NEWTYPE(cmd, struct ifdrv);
break;
case SIOCAIFGROUP32:
case SIOCGIFGROUP32:
case SIOCDIFGROUP32:
case SIOCGIFGMEMB32:
ifgr32 = (struct ifgroupreq32 *)data;
memcpy(thunk.ifgr.ifgr_name, ifgr32->ifgr_name,
sizeof(thunk.ifgr.ifgr_name));
thunk.ifgr.ifgr_len = ifgr32->ifgr_len;
switch (cmd) {
case SIOCAIFGROUP32:
case SIOCDIFGROUP32:
memcpy(thunk.ifgr.ifgr_group, ifgr32->ifgr_group,
sizeof(thunk.ifgr.ifgr_group));
break;
case SIOCGIFGROUP32:
case SIOCGIFGMEMB32:
thunk.ifgr.ifgr_groups = PTRIN(ifgr32->ifgr_groups);
break;
}
data = (caddr_t)&thunk.ifgr;
cmd = _IOC_NEWTYPE(cmd, struct ifgroupreq);
break;
case SIOCGIFMEDIA32:
case SIOCGIFXMEDIA32:
ifmr32 = (struct ifmediareq32 *)data;
memcpy(thunk.ifmr.ifm_name, ifmr32->ifm_name,
sizeof(thunk.ifmr.ifm_name));
thunk.ifmr.ifm_current = ifmr32->ifm_current;
thunk.ifmr.ifm_mask = ifmr32->ifm_mask;
thunk.ifmr.ifm_status = ifmr32->ifm_status;
thunk.ifmr.ifm_active = ifmr32->ifm_active;
thunk.ifmr.ifm_count = ifmr32->ifm_count;
thunk.ifmr.ifm_ulist = PTRIN(ifmr32->ifm_ulist);
data = (caddr_t)&thunk.ifmr;
cmd = _IOC_NEWTYPE(cmd, struct ifmediareq);
break;
}
#endif
switch (cmd) {
case SIOCGIFCONF:
error = ifconf(cmd, data);
goto out_noref;
}
ifr = (struct ifreq *)data;
switch (cmd) {
#ifdef VIMAGE
case SIOCSIFRVNET:
error = priv_check(td, PRIV_NET_SETIFVNET);
if (error == 0)
error = if_vmove_reclaim(td, ifr->ifr_name,
ifr->ifr_jid);
goto out_noref;
#endif
case SIOCIFCREATE:
case SIOCIFCREATE2:
error = priv_check(td, PRIV_NET_IFCREATE);
if (error == 0)
error = if_clone_create(ifr->ifr_name,
sizeof(ifr->ifr_name), cmd == SIOCIFCREATE2 ?
ifr_data_get_ptr(ifr) : NULL);
goto out_noref;
case SIOCIFDESTROY:
error = priv_check(td, PRIV_NET_IFDESTROY);
if (error == 0) {
sx_xlock(&ifnet_detach_sxlock);
error = if_clone_destroy(ifr->ifr_name);
sx_xunlock(&ifnet_detach_sxlock);
}
goto out_noref;
case SIOCIFGCLONERS:
error = if_clone_list((struct if_clonereq *)data);
goto out_noref;
case SIOCGIFGMEMB:
error = if_getgroupmembers((struct ifgroupreq *)data);
goto out_noref;
#if defined(INET) || defined(INET6)
case SIOCSVH:
case SIOCGVH:
if (carp_ioctl_p == NULL)
error = EPROTONOSUPPORT;
else
error = (*carp_ioctl_p)(ifr, cmd, td);
goto out_noref;
#endif
}
ifp = ifunit_ref(ifr->ifr_name);
if (ifp == NULL) {
error = ENXIO;
goto out_noref;
}
error = ifhwioctl(cmd, ifp, data, td);
if (error != ENOIOCTL)
goto out_ref;
oif_flags = ifp->if_flags;
if (so->so_proto == NULL) {
error = EOPNOTSUPP;
goto out_ref;
}
/*
* Pass the request on to the socket control method, and if the
* latter returns EOPNOTSUPP, directly to the interface.
*
* Make an exception for the legacy SIOCSIF* requests. Drivers
* trust SIOCSIFADDR et al to come from an already privileged
* layer, and do not perform any credentials checks or input
* validation.
*/
error = ((*so->so_proto->pr_usrreqs->pru_control)(so, cmd, data,
ifp, td));
if (error == EOPNOTSUPP && ifp != NULL && ifp->if_ioctl != NULL &&
cmd != SIOCSIFADDR && cmd != SIOCSIFBRDADDR &&
cmd != SIOCSIFDSTADDR && cmd != SIOCSIFNETMASK)
error = (*ifp->if_ioctl)(ifp, cmd, data);
if ((oif_flags ^ ifp->if_flags) & IFF_UP) {
#ifdef INET6
if (ifp->if_flags & IFF_UP)
in6_if_up(ifp);
#endif
}
out_ref:
if_rele(ifp);
out_noref:
CURVNET_RESTORE();
#ifdef COMPAT_FREEBSD32
if (error != 0)
return (error);
switch (saved_cmd) {
case SIOCGIFCONF32:
ifc32->ifc_len = thunk.ifc.ifc_len;
break;
case SIOCGDRVSPEC32:
/*
* SIOCGDRVSPEC is IOWR, but nothing actually touches
* the struct so just assert that ifd_len (the only
* field it might make sense to update) hasn't
* changed.
*/
KASSERT(thunk.ifd.ifd_len == ifd32->ifd_len,
("ifd_len was updated %u -> %zu", ifd32->ifd_len,
thunk.ifd.ifd_len));
break;
case SIOCGIFGROUP32:
case SIOCGIFGMEMB32:
ifgr32->ifgr_len = thunk.ifgr.ifgr_len;
break;
case SIOCGIFMEDIA32:
case SIOCGIFXMEDIA32:
ifmr32->ifm_current = thunk.ifmr.ifm_current;
ifmr32->ifm_mask = thunk.ifmr.ifm_mask;
ifmr32->ifm_status = thunk.ifmr.ifm_status;
ifmr32->ifm_active = thunk.ifmr.ifm_active;
ifmr32->ifm_count = thunk.ifmr.ifm_count;
break;
}
#endif
return (error);
}
/*
* The code common to handling reference counted flags,
* e.g., in ifpromisc() and if_allmulti().
* The "pflag" argument can specify a permanent mode flag to check,
* such as IFF_PPROMISC for promiscuous mode; should be 0 if none.
*
* Only to be used on stack-owned flags, not driver-owned flags.
*/
static int
if_setflag(struct ifnet *ifp, int flag, int pflag, int *refcount, int onswitch)
{
struct ifreq ifr;
int error;
int oldflags, oldcount;
/* Sanity checks to catch programming errors */
KASSERT((flag & (IFF_DRV_OACTIVE|IFF_DRV_RUNNING)) == 0,
("%s: setting driver-owned flag %d", __func__, flag));
if (onswitch)
KASSERT(*refcount >= 0,
("%s: increment negative refcount %d for flag %d",
__func__, *refcount, flag));
else
KASSERT(*refcount > 0,
("%s: decrement non-positive refcount %d for flag %d",
__func__, *refcount, flag));
/* In case this mode is permanent, just touch refcount */
if (ifp->if_flags & pflag) {
*refcount += onswitch ? 1 : -1;
return (0);
}
/* Save ifnet parameters for if_ioctl() may fail */
oldcount = *refcount;
oldflags = ifp->if_flags;
/*
* See if we aren't the only and touching refcount is enough.
* Actually toggle interface flag if we are the first or last.
*/
if (onswitch) {
if ((*refcount)++)
return (0);
ifp->if_flags |= flag;
} else {
if (--(*refcount))
return (0);
ifp->if_flags &= ~flag;
}
/* Call down the driver since we've changed interface flags */
if (ifp->if_ioctl == NULL) {
error = EOPNOTSUPP;
goto recover;
}
ifr.ifr_flags = ifp->if_flags & 0xffff;
ifr.ifr_flagshigh = ifp->if_flags >> 16;
error = (*ifp->if_ioctl)(ifp, SIOCSIFFLAGS, (caddr_t)&ifr);
if (error)
goto recover;
/* Notify userland that interface flags have changed */
rt_ifmsg(ifp);
return (0);
recover:
/* Recover after driver error */
*refcount = oldcount;
ifp->if_flags = oldflags;
return (error);
}
/*
* Set/clear promiscuous mode on interface ifp based on the truth value
* of pswitch. The calls are reference counted so that only the first
* "on" request actually has an effect, as does the final "off" request.
* Results are undefined if the "off" and "on" requests are not matched.
*/
int
ifpromisc(struct ifnet *ifp, int pswitch)
{
int error;
int oldflags = ifp->if_flags;
error = if_setflag(ifp, IFF_PROMISC, IFF_PPROMISC,
&ifp->if_pcount, pswitch);
/* If promiscuous mode status has changed, log a message */
if (error == 0 && ((ifp->if_flags ^ oldflags) & IFF_PROMISC) &&
log_promisc_mode_change)
if_printf(ifp, "promiscuous mode %s\n",
(ifp->if_flags & IFF_PROMISC) ? "enabled" : "disabled");
return (error);
}
/*
* Return interface configuration
* of system. List may be used
* in later ioctl's (above) to get
* other information.
*/
/*ARGSUSED*/
static int
ifconf(u_long cmd, caddr_t data)
{
struct ifconf *ifc = (struct ifconf *)data;
struct ifnet *ifp;
struct ifaddr *ifa;
struct ifreq ifr;
struct sbuf *sb;
int error, full = 0, valid_len, max_len;
/* Limit initial buffer size to maxphys to avoid DoS from userspace. */
max_len = maxphys - 1;
/* Prevent hostile input from being able to crash the system */
if (ifc->ifc_len <= 0)
return (EINVAL);
again:
if (ifc->ifc_len <= max_len) {
max_len = ifc->ifc_len;
full = 1;
}
sb = sbuf_new(NULL, NULL, max_len + 1, SBUF_FIXEDLEN);
max_len = 0;
valid_len = 0;
IFNET_RLOCK();
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
struct epoch_tracker et;
int addrs;
/*
* Zero the ifr to make sure we don't disclose the contents
* of the stack.
*/
memset(&ifr, 0, sizeof(ifr));
if (strlcpy(ifr.ifr_name, ifp->if_xname, sizeof(ifr.ifr_name))
>= sizeof(ifr.ifr_name)) {
sbuf_delete(sb);
IFNET_RUNLOCK();
return (ENAMETOOLONG);
}
addrs = 0;
NET_EPOCH_ENTER(et);
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
struct sockaddr *sa = ifa->ifa_addr;
if (prison_if(curthread->td_ucred, sa) != 0)
continue;
addrs++;
if (sa->sa_len <= sizeof(*sa)) {
if (sa->sa_len < sizeof(*sa)) {
memset(&ifr.ifr_ifru.ifru_addr, 0,
sizeof(ifr.ifr_ifru.ifru_addr));
memcpy(&ifr.ifr_ifru.ifru_addr, sa,
sa->sa_len);
} else
ifr.ifr_ifru.ifru_addr = *sa;
sbuf_bcat(sb, &ifr, sizeof(ifr));
max_len += sizeof(ifr);
} else {
sbuf_bcat(sb, &ifr,
offsetof(struct ifreq, ifr_addr));
max_len += offsetof(struct ifreq, ifr_addr);
sbuf_bcat(sb, sa, sa->sa_len);
max_len += sa->sa_len;
}
if (sbuf_error(sb) == 0)
valid_len = sbuf_len(sb);
}
NET_EPOCH_EXIT(et);
if (addrs == 0) {
sbuf_bcat(sb, &ifr, sizeof(ifr));
max_len += sizeof(ifr);
if (sbuf_error(sb) == 0)
valid_len = sbuf_len(sb);
}
}
IFNET_RUNLOCK();
/*
* If we didn't allocate enough space (uncommon), try again. If
* we have already allocated as much space as we are allowed,
* return what we've got.
*/
if (valid_len != max_len && !full) {
sbuf_delete(sb);
goto again;
}
ifc->ifc_len = valid_len;
sbuf_finish(sb);
error = copyout(sbuf_data(sb), ifc->ifc_req, ifc->ifc_len);
sbuf_delete(sb);
return (error);
}
/*
* Just like ifpromisc(), but for all-multicast-reception mode.
*/
int
if_allmulti(struct ifnet *ifp, int onswitch)
{
return (if_setflag(ifp, IFF_ALLMULTI, 0, &ifp->if_amcount, onswitch));
}
struct ifmultiaddr *
if_findmulti(struct ifnet *ifp, const struct sockaddr *sa)
{
struct ifmultiaddr *ifma;
IF_ADDR_LOCK_ASSERT(ifp);
CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
if (sa->sa_family == AF_LINK) {
if (sa_dl_equal(ifma->ifma_addr, sa))
break;
} else {
if (sa_equal(ifma->ifma_addr, sa))
break;
}
}
return ifma;
}
/*
* Allocate a new ifmultiaddr and initialize based on passed arguments. We
* make copies of passed sockaddrs. The ifmultiaddr will not be added to
* the ifnet multicast address list here, so the caller must do that and
* other setup work (such as notifying the device driver). The reference
* count is initialized to 1.
*/
static struct ifmultiaddr *
if_allocmulti(struct ifnet *ifp, struct sockaddr *sa, struct sockaddr *llsa,
int mflags)
{
struct ifmultiaddr *ifma;
struct sockaddr *dupsa;
ifma = malloc(sizeof *ifma, M_IFMADDR, mflags |
M_ZERO);
if (ifma == NULL)
return (NULL);
dupsa = malloc(sa->sa_len, M_IFMADDR, mflags);
if (dupsa == NULL) {
free(ifma, M_IFMADDR);
return (NULL);
}
bcopy(sa, dupsa, sa->sa_len);
ifma->ifma_addr = dupsa;
ifma->ifma_ifp = ifp;
ifma->ifma_refcount = 1;
ifma->ifma_protospec = NULL;
if (llsa == NULL) {
ifma->ifma_lladdr = NULL;
return (ifma);
}
dupsa = malloc(llsa->sa_len, M_IFMADDR, mflags);
if (dupsa == NULL) {
free(ifma->ifma_addr, M_IFMADDR);
free(ifma, M_IFMADDR);
return (NULL);
}
bcopy(llsa, dupsa, llsa->sa_len);
ifma->ifma_lladdr = dupsa;
return (ifma);
}
/*
* if_freemulti: free ifmultiaddr structure and possibly attached related
* addresses. The caller is responsible for implementing reference
* counting, notifying the driver, handling routing messages, and releasing
* any dependent link layer state.
*/
#ifdef MCAST_VERBOSE
extern void kdb_backtrace(void);
#endif
static void
if_freemulti_internal(struct ifmultiaddr *ifma)
{
KASSERT(ifma->ifma_refcount == 0, ("if_freemulti: refcount %d",
ifma->ifma_refcount));
if (ifma->ifma_lladdr != NULL)
free(ifma->ifma_lladdr, M_IFMADDR);
#ifdef MCAST_VERBOSE
kdb_backtrace();
printf("%s freeing ifma: %p\n", __func__, ifma);
#endif
free(ifma->ifma_addr, M_IFMADDR);
free(ifma, M_IFMADDR);
}
static void
if_destroymulti(epoch_context_t ctx)
{
struct ifmultiaddr *ifma;
ifma = __containerof(ctx, struct ifmultiaddr, ifma_epoch_ctx);
if_freemulti_internal(ifma);
}
void
if_freemulti(struct ifmultiaddr *ifma)
{
KASSERT(ifma->ifma_refcount == 0, ("if_freemulti_epoch: refcount %d",
ifma->ifma_refcount));
NET_EPOCH_CALL(if_destroymulti, &ifma->ifma_epoch_ctx);
}
/*
* Register an additional multicast address with a network interface.
*
* - If the address is already present, bump the reference count on the
* address and return.
* - If the address is not link-layer, look up a link layer address.
* - Allocate address structures for one or both addresses, and attach to the
* multicast address list on the interface. If automatically adding a link
* layer address, the protocol address will own a reference to the link
* layer address, to be freed when it is freed.
* - Notify the network device driver of an addition to the multicast address
* list.
*
* 'sa' points to caller-owned memory with the desired multicast address.
*
* 'retifma' will be used to return a pointer to the resulting multicast
* address reference, if desired.
*/
int
if_addmulti(struct ifnet *ifp, struct sockaddr *sa,
struct ifmultiaddr **retifma)
{
struct ifmultiaddr *ifma, *ll_ifma;
struct sockaddr *llsa;
struct sockaddr_dl sdl;
int error;
#ifdef INET
IN_MULTI_LIST_UNLOCK_ASSERT();
#endif
#ifdef INET6
IN6_MULTI_LIST_UNLOCK_ASSERT();
#endif
/*
* If the address is already present, return a new reference to it;
* otherwise, allocate storage and set up a new address.
*/
IF_ADDR_WLOCK(ifp);
ifma = if_findmulti(ifp, sa);
if (ifma != NULL) {
ifma->ifma_refcount++;
if (retifma != NULL)
*retifma = ifma;
IF_ADDR_WUNLOCK(ifp);
return (0);
}
/*
* The address isn't already present; resolve the protocol address
* into a link layer address, and then look that up, bump its
* refcount or allocate an ifma for that also.
* Most link layer resolving functions returns address data which
* fits inside default sockaddr_dl structure. However callback
* can allocate another sockaddr structure, in that case we need to
* free it later.
*/
llsa = NULL;
ll_ifma = NULL;
if (ifp->if_resolvemulti != NULL) {
/* Provide called function with buffer size information */
sdl.sdl_len = sizeof(sdl);
llsa = (struct sockaddr *)&sdl;
error = ifp->if_resolvemulti(ifp, &llsa, sa);
if (error)
goto unlock_out;
}
/*
* Allocate the new address. Don't hook it up yet, as we may also
* need to allocate a link layer multicast address.
*/
ifma = if_allocmulti(ifp, sa, llsa, M_NOWAIT);
if (ifma == NULL) {
error = ENOMEM;
goto free_llsa_out;
}
/*
* If a link layer address is found, we'll need to see if it's
* already present in the address list, or allocate is as well.
* When this block finishes, the link layer address will be on the
* list.
*/
if (llsa != NULL) {
ll_ifma = if_findmulti(ifp, llsa);
if (ll_ifma == NULL) {
ll_ifma = if_allocmulti(ifp, llsa, NULL, M_NOWAIT);
if (ll_ifma == NULL) {
--ifma->ifma_refcount;
if_freemulti(ifma);
error = ENOMEM;
goto free_llsa_out;
}
ll_ifma->ifma_flags |= IFMA_F_ENQUEUED;
CK_STAILQ_INSERT_HEAD(&ifp->if_multiaddrs, ll_ifma,
ifma_link);
} else
ll_ifma->ifma_refcount++;
ifma->ifma_llifma = ll_ifma;
}
/*
* We now have a new multicast address, ifma, and possibly a new or
* referenced link layer address. Add the primary address to the
* ifnet address list.
*/
ifma->ifma_flags |= IFMA_F_ENQUEUED;
CK_STAILQ_INSERT_HEAD(&ifp->if_multiaddrs, ifma, ifma_link);
if (retifma != NULL)
*retifma = ifma;
/*
* Must generate the message while holding the lock so that 'ifma'
* pointer is still valid.
*/
rt_newmaddrmsg(RTM_NEWMADDR, ifma);
IF_ADDR_WUNLOCK(ifp);
/*
* We are certain we have added something, so call down to the
* interface to let them know about it.
*/
if (ifp->if_ioctl != NULL) {
if (THREAD_CAN_SLEEP())
(void )(*ifp->if_ioctl)(ifp, SIOCADDMULTI, 0);
else
taskqueue_enqueue(taskqueue_swi, &ifp->if_addmultitask);
}
if ((llsa != NULL) && (llsa != (struct sockaddr *)&sdl))
link_free_sdl(llsa);
return (0);
free_llsa_out:
if ((llsa != NULL) && (llsa != (struct sockaddr *)&sdl))
link_free_sdl(llsa);
unlock_out:
IF_ADDR_WUNLOCK(ifp);
return (error);
}
static void
if_siocaddmulti(void *arg, int pending)
{
struct ifnet *ifp;
ifp = arg;
#ifdef DIAGNOSTIC
if (pending > 1)
if_printf(ifp, "%d SIOCADDMULTI coalesced\n", pending);
#endif
CURVNET_SET(ifp->if_vnet);
(void )(*ifp->if_ioctl)(ifp, SIOCADDMULTI, 0);
CURVNET_RESTORE();
}
/*
* Delete a multicast group membership by network-layer group address.
*
* Returns ENOENT if the entry could not be found. If ifp no longer
* exists, results are undefined. This entry point should only be used
* from subsystems which do appropriate locking to hold ifp for the
* duration of the call.
* Network-layer protocol domains must use if_delmulti_ifma().
*/
int
if_delmulti(struct ifnet *ifp, struct sockaddr *sa)
{
struct ifmultiaddr *ifma;
int lastref;
KASSERT(ifp, ("%s: NULL ifp", __func__));
IF_ADDR_WLOCK(ifp);
lastref = 0;
ifma = if_findmulti(ifp, sa);
if (ifma != NULL)
lastref = if_delmulti_locked(ifp, ifma, 0);
IF_ADDR_WUNLOCK(ifp);
if (ifma == NULL)
return (ENOENT);
if (lastref && ifp->if_ioctl != NULL) {
(void)(*ifp->if_ioctl)(ifp, SIOCDELMULTI, 0);
}
return (0);
}
/*
* Delete all multicast group membership for an interface.
* Should be used to quickly flush all multicast filters.
*/
void
if_delallmulti(struct ifnet *ifp)
{
struct ifmultiaddr *ifma;
struct ifmultiaddr *next;
IF_ADDR_WLOCK(ifp);
CK_STAILQ_FOREACH_SAFE(ifma, &ifp->if_multiaddrs, ifma_link, next)
if_delmulti_locked(ifp, ifma, 0);
IF_ADDR_WUNLOCK(ifp);
}
void
if_delmulti_ifma(struct ifmultiaddr *ifma)
{
if_delmulti_ifma_flags(ifma, 0);
}
/*
* Delete a multicast group membership by group membership pointer.
* Network-layer protocol domains must use this routine.
*
* It is safe to call this routine if the ifp disappeared.
*/
void
if_delmulti_ifma_flags(struct ifmultiaddr *ifma, int flags)
{
struct ifnet *ifp;
int lastref;
MCDPRINTF("%s freeing ifma: %p\n", __func__, ifma);
#ifdef INET
IN_MULTI_LIST_UNLOCK_ASSERT();
#endif
ifp = ifma->ifma_ifp;
#ifdef DIAGNOSTIC
if (ifp == NULL) {
printf("%s: ifma_ifp seems to be detached\n", __func__);
} else {
struct epoch_tracker et;
struct ifnet *oifp;
NET_EPOCH_ENTER(et);
CK_STAILQ_FOREACH(oifp, &V_ifnet, if_link)
if (ifp == oifp)
break;
NET_EPOCH_EXIT(et);
if (ifp != oifp)
ifp = NULL;
}
#endif
/*
* If and only if the ifnet instance exists: Acquire the address lock.
*/
if (ifp != NULL)
IF_ADDR_WLOCK(ifp);
lastref = if_delmulti_locked(ifp, ifma, flags);
if (ifp != NULL) {
/*
* If and only if the ifnet instance exists:
* Release the address lock.
* If the group was left: update the hardware hash filter.
*/
IF_ADDR_WUNLOCK(ifp);
if (lastref && ifp->if_ioctl != NULL) {
(void)(*ifp->if_ioctl)(ifp, SIOCDELMULTI, 0);
}
}
}
/*
* Perform deletion of network-layer and/or link-layer multicast address.
*
* Return 0 if the reference count was decremented.
* Return 1 if the final reference was released, indicating that the
* hardware hash filter should be reprogrammed.
*/
static int
if_delmulti_locked(struct ifnet *ifp, struct ifmultiaddr *ifma, int detaching)
{
struct ifmultiaddr *ll_ifma;
if (ifp != NULL && ifma->ifma_ifp != NULL) {
KASSERT(ifma->ifma_ifp == ifp,
("%s: inconsistent ifp %p", __func__, ifp));
IF_ADDR_WLOCK_ASSERT(ifp);
}
ifp = ifma->ifma_ifp;
MCDPRINTF("%s freeing %p from %s \n", __func__, ifma, ifp ? ifp->if_xname : "");
/*
* If the ifnet is detaching, null out references to ifnet,
* so that upper protocol layers will notice, and not attempt
* to obtain locks for an ifnet which no longer exists. The
* routing socket announcement must happen before the ifnet
* instance is detached from the system.
*/
if (detaching) {
#ifdef DIAGNOSTIC
printf("%s: detaching ifnet instance %p\n", __func__, ifp);
#endif
/*
* ifp may already be nulled out if we are being reentered
* to delete the ll_ifma.
*/
if (ifp != NULL) {
rt_newmaddrmsg(RTM_DELMADDR, ifma);
ifma->ifma_ifp = NULL;
}
}
if (--ifma->ifma_refcount > 0)
return 0;
if (ifp != NULL && detaching == 0 && (ifma->ifma_flags & IFMA_F_ENQUEUED)) {
CK_STAILQ_REMOVE(&ifp->if_multiaddrs, ifma, ifmultiaddr, ifma_link);
ifma->ifma_flags &= ~IFMA_F_ENQUEUED;
}
/*
* If this ifma is a network-layer ifma, a link-layer ifma may
* have been associated with it. Release it first if so.
*/
ll_ifma = ifma->ifma_llifma;
if (ll_ifma != NULL) {
KASSERT(ifma->ifma_lladdr != NULL,
("%s: llifma w/o lladdr", __func__));
if (detaching)
ll_ifma->ifma_ifp = NULL; /* XXX */
if (--ll_ifma->ifma_refcount == 0) {
if (ifp != NULL) {
if (ll_ifma->ifma_flags & IFMA_F_ENQUEUED) {
CK_STAILQ_REMOVE(&ifp->if_multiaddrs, ll_ifma, ifmultiaddr,
ifma_link);
ll_ifma->ifma_flags &= ~IFMA_F_ENQUEUED;
}
}
if_freemulti(ll_ifma);
}
}
#ifdef INVARIANTS
if (ifp) {
struct ifmultiaddr *ifmatmp;
CK_STAILQ_FOREACH(ifmatmp, &ifp->if_multiaddrs, ifma_link)
MPASS(ifma != ifmatmp);
}
#endif
if_freemulti(ifma);
/*
* The last reference to this instance of struct ifmultiaddr
* was released; the hardware should be notified of this change.
*/
return 1;
}
/*
* Set the link layer address on an interface.
*
* At this time we only support certain types of interfaces,
* and we don't allow the length of the address to change.
*
* Set noinline to be dtrace-friendly
*/
__noinline int
if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len)
{
struct sockaddr_dl *sdl;
struct ifaddr *ifa;
struct ifreq ifr;
ifa = ifp->if_addr;
if (ifa == NULL)
return (EINVAL);
sdl = (struct sockaddr_dl *)ifa->ifa_addr;
if (sdl == NULL)
return (EINVAL);
if (len != sdl->sdl_alen) /* don't allow length to change */
return (EINVAL);
switch (ifp->if_type) {
case IFT_ETHER:
case IFT_XETHER:
case IFT_L2VLAN:
case IFT_BRIDGE:
case IFT_IEEE8023ADLAG:
bcopy(lladdr, LLADDR(sdl), len);
break;
default:
return (ENODEV);
}
/*
* If the interface is already up, we need
* to re-init it in order to reprogram its
* address filter.
*/
if ((ifp->if_flags & IFF_UP) != 0) {
if (ifp->if_ioctl) {
ifp->if_flags &= ~IFF_UP;
ifr.ifr_flags = ifp->if_flags & 0xffff;
ifr.ifr_flagshigh = ifp->if_flags >> 16;
(*ifp->if_ioctl)(ifp, SIOCSIFFLAGS, (caddr_t)&ifr);
ifp->if_flags |= IFF_UP;
ifr.ifr_flags = ifp->if_flags & 0xffff;
ifr.ifr_flagshigh = ifp->if_flags >> 16;
(*ifp->if_ioctl)(ifp, SIOCSIFFLAGS, (caddr_t)&ifr);
}
}
EVENTHANDLER_INVOKE(iflladdr_event, ifp);
return (0);
}
/*
* Compat function for handling basic encapsulation requests.
* Not converted stacks (FDDI, IB, ..) supports traditional
* output model: ARP (and other similar L2 protocols) are handled
* inside output routine, arpresolve/nd6_resolve() returns MAC
* address instead of full prepend.
*
* This function creates calculated header==MAC for IPv4/IPv6 and
* returns EAFNOSUPPORT (which is then handled in ARP code) for other
* address families.
*/
static int
if_requestencap_default(struct ifnet *ifp, struct if_encap_req *req)
{
if (req->rtype != IFENCAP_LL)
return (EOPNOTSUPP);
if (req->bufsize < req->lladdr_len)
return (ENOMEM);
switch (req->family) {
case AF_INET:
case AF_INET6:
break;
default:
return (EAFNOSUPPORT);
}
/* Copy lladdr to storage as is */
memmove(req->buf, req->lladdr, req->lladdr_len);
req->bufsize = req->lladdr_len;
req->lladdr_off = 0;
return (0);
}
/*
* Tunnel interfaces can nest, also they may cause infinite recursion
* calls when misconfigured. We'll prevent this by detecting loops.
* High nesting level may cause stack exhaustion. We'll prevent this
* by introducing upper limit.
*
* Return 0, if tunnel nesting count is equal or less than limit.
*/
int
if_tunnel_check_nesting(struct ifnet *ifp, struct mbuf *m, uint32_t cookie,
int limit)
{
struct m_tag *mtag;
int count;
count = 1;
mtag = NULL;
while ((mtag = m_tag_locate(m, cookie, 0, mtag)) != NULL) {
if (*(struct ifnet **)(mtag + 1) == ifp) {
log(LOG_NOTICE, "%s: loop detected\n", if_name(ifp));
return (EIO);
}
count++;
}
if (count > limit) {
log(LOG_NOTICE,
"%s: if_output recursively called too many times(%d)\n",
if_name(ifp), count);
return (EIO);
}
mtag = m_tag_alloc(cookie, 0, sizeof(struct ifnet *), M_NOWAIT);
if (mtag == NULL)
return (ENOMEM);
*(struct ifnet **)(mtag + 1) = ifp;
m_tag_prepend(m, mtag);
return (0);
}
/*
* Get the link layer address that was read from the hardware at attach.
*
* This is only set by Ethernet NICs (IFT_ETHER), but laggX interfaces re-type
* their component interfaces as IFT_IEEE8023ADLAG.
*/
int
if_gethwaddr(struct ifnet *ifp, struct ifreq *ifr)
{
if (ifp->if_hw_addr == NULL)
return (ENODEV);
switch (ifp->if_type) {
case IFT_ETHER:
case IFT_IEEE8023ADLAG:
bcopy(ifp->if_hw_addr, ifr->ifr_addr.sa_data, ifp->if_addrlen);
return (0);
default:
return (ENODEV);
}
}
/*
* The name argument must be a pointer to storage which will last as
* long as the interface does. For physical devices, the result of
* device_get_name(dev) is a good choice and for pseudo-devices a
* static string works well.
*/
void
if_initname(struct ifnet *ifp, const char *name, int unit)
{
ifp->if_dname = name;
ifp->if_dunit = unit;
if (unit != IF_DUNIT_NONE)
snprintf(ifp->if_xname, IFNAMSIZ, "%s%d", name, unit);
else
strlcpy(ifp->if_xname, name, IFNAMSIZ);
}
static int
if_vlog(struct ifnet *ifp, int pri, const char *fmt, va_list ap)
{
char if_fmt[256];
snprintf(if_fmt, sizeof(if_fmt), "%s: %s", ifp->if_xname, fmt);
vlog(pri, if_fmt, ap);
return (0);
}
int
if_printf(struct ifnet *ifp, const char *fmt, ...)
{
va_list ap;
va_start(ap, fmt);
if_vlog(ifp, LOG_INFO, fmt, ap);
va_end(ap);
return (0);
}
int
if_log(struct ifnet *ifp, int pri, const char *fmt, ...)
{
va_list ap;
va_start(ap, fmt);
if_vlog(ifp, pri, fmt, ap);
va_end(ap);
return (0);
}
void
if_start(struct ifnet *ifp)
{
(*(ifp)->if_start)(ifp);
}
/*
* Backwards compatibility interface for drivers
* that have not implemented it
*/
static int
if_transmit(struct ifnet *ifp, struct mbuf *m)
{
int error;
IFQ_HANDOFF(ifp, m, error);
return (error);
}
static void
if_input_default(struct ifnet *ifp __unused, struct mbuf *m)
{
m_freem(m);
}
int
if_handoff(struct ifqueue *ifq, struct mbuf *m, struct ifnet *ifp, int adjust)
{
int active = 0;
IF_LOCK(ifq);
if (_IF_QFULL(ifq)) {
IF_UNLOCK(ifq);
if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1);
m_freem(m);
return (0);
}
if (ifp != NULL) {
if_inc_counter(ifp, IFCOUNTER_OBYTES, m->m_pkthdr.len + adjust);
if (m->m_flags & (M_BCAST|M_MCAST))
if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
active = ifp->if_drv_flags & IFF_DRV_OACTIVE;
}
_IF_ENQUEUE(ifq, m);
IF_UNLOCK(ifq);
if (ifp != NULL && !active)
(*(ifp)->if_start)(ifp);
return (1);
}
void
if_register_com_alloc(u_char type,
if_com_alloc_t *a, if_com_free_t *f)
{
KASSERT(if_com_alloc[type] == NULL,
("if_register_com_alloc: %d already registered", type));
KASSERT(if_com_free[type] == NULL,
("if_register_com_alloc: %d free already registered", type));
if_com_alloc[type] = a;
if_com_free[type] = f;
}
void
if_deregister_com_alloc(u_char type)
{
KASSERT(if_com_alloc[type] != NULL,
("if_deregister_com_alloc: %d not registered", type));
KASSERT(if_com_free[type] != NULL,
("if_deregister_com_alloc: %d free not registered", type));
/*
* Ensure all pending EPOCH(9) callbacks have been executed. This
* fixes issues about late invocation of if_destroy(), which leads
* to memory leak from if_com_alloc[type] allocated if_l2com.
*/
epoch_drain_callbacks(net_epoch_preempt);
if_com_alloc[type] = NULL;
if_com_free[type] = NULL;
}
/* API for driver access to network stack owned ifnet.*/
uint64_t
if_setbaudrate(struct ifnet *ifp, uint64_t baudrate)
{
uint64_t oldbrate;
oldbrate = ifp->if_baudrate;
ifp->if_baudrate = baudrate;
return (oldbrate);
}
uint64_t
if_getbaudrate(if_t ifp)
{
return (((struct ifnet *)ifp)->if_baudrate);
}
int
if_setcapabilities(if_t ifp, int capabilities)
{
((struct ifnet *)ifp)->if_capabilities = capabilities;
return (0);
}
int
if_setcapabilitiesbit(if_t ifp, int setbit, int clearbit)
{
((struct ifnet *)ifp)->if_capabilities |= setbit;
((struct ifnet *)ifp)->if_capabilities &= ~clearbit;
return (0);
}
int
if_getcapabilities(if_t ifp)
{
return ((struct ifnet *)ifp)->if_capabilities;
}
int
if_setcapenable(if_t ifp, int capabilities)
{
((struct ifnet *)ifp)->if_capenable = capabilities;
return (0);
}
int
if_setcapenablebit(if_t ifp, int setcap, int clearcap)
{
if(setcap)
((struct ifnet *)ifp)->if_capenable |= setcap;
if(clearcap)
((struct ifnet *)ifp)->if_capenable &= ~clearcap;
return (0);
}
const char *
if_getdname(if_t ifp)
{
return ((struct ifnet *)ifp)->if_dname;
}
int
if_togglecapenable(if_t ifp, int togglecap)
{
((struct ifnet *)ifp)->if_capenable ^= togglecap;
return (0);
}
int
if_getcapenable(if_t ifp)
{
return ((struct ifnet *)ifp)->if_capenable;
}
/*
* This is largely undesirable because it ties ifnet to a device, but does
* provide flexiblity for an embedded product vendor. Should be used with
* the understanding that it violates the interface boundaries, and should be
* a last resort only.
*/
int
if_setdev(if_t ifp, void *dev)
{
return (0);
}
int
if_setdrvflagbits(if_t ifp, int set_flags, int clear_flags)
{
((struct ifnet *)ifp)->if_drv_flags |= set_flags;
((struct ifnet *)ifp)->if_drv_flags &= ~clear_flags;
return (0);
}
int
if_getdrvflags(if_t ifp)
{
return ((struct ifnet *)ifp)->if_drv_flags;
}
int
if_setdrvflags(if_t ifp, int flags)
{
((struct ifnet *)ifp)->if_drv_flags = flags;
return (0);
}
int
if_setflags(if_t ifp, int flags)
{
ifp->if_flags = flags;
return (0);
}
int
if_setflagbits(if_t ifp, int set, int clear)
{
((struct ifnet *)ifp)->if_flags |= set;
((struct ifnet *)ifp)->if_flags &= ~clear;
return (0);
}
int
if_getflags(if_t ifp)
{
return ((struct ifnet *)ifp)->if_flags;
}
int
if_clearhwassist(if_t ifp)
{
((struct ifnet *)ifp)->if_hwassist = 0;
return (0);
}
int
if_sethwassistbits(if_t ifp, int toset, int toclear)
{
((struct ifnet *)ifp)->if_hwassist |= toset;
((struct ifnet *)ifp)->if_hwassist &= ~toclear;
return (0);
}
int
if_sethwassist(if_t ifp, int hwassist_bit)
{
((struct ifnet *)ifp)->if_hwassist = hwassist_bit;
return (0);
}
int
if_gethwassist(if_t ifp)
{
return ((struct ifnet *)ifp)->if_hwassist;
}
int
if_setmtu(if_t ifp, int mtu)
{
((struct ifnet *)ifp)->if_mtu = mtu;
return (0);
}
int
if_getmtu(if_t ifp)
{
return ((struct ifnet *)ifp)->if_mtu;
}
int
if_getmtu_family(if_t ifp, int family)
{
struct domain *dp;
for (dp = domains; dp; dp = dp->dom_next) {
if (dp->dom_family == family && dp->dom_ifmtu != NULL)
return (dp->dom_ifmtu((struct ifnet *)ifp));
}
return (((struct ifnet *)ifp)->if_mtu);
}
/*
* Methods for drivers to access interface unicast and multicast
* link level addresses. Driver shall not know 'struct ifaddr' neither
* 'struct ifmultiaddr'.
*/
u_int
if_lladdr_count(if_t ifp)
{
struct epoch_tracker et;
struct ifaddr *ifa;
u_int count;
count = 0;
NET_EPOCH_ENTER(et);
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link)
if (ifa->ifa_addr->sa_family == AF_LINK)
count++;
NET_EPOCH_EXIT(et);
return (count);
}
u_int
if_foreach_lladdr(if_t ifp, iflladdr_cb_t cb, void *cb_arg)
{
struct epoch_tracker et;
struct ifaddr *ifa;
u_int count;
MPASS(cb);
count = 0;
NET_EPOCH_ENTER(et);
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family != AF_LINK)
continue;
count += (*cb)(cb_arg, (struct sockaddr_dl *)ifa->ifa_addr,
count);
}
NET_EPOCH_EXIT(et);
return (count);
}
u_int
if_llmaddr_count(if_t ifp)
{
struct epoch_tracker et;
struct ifmultiaddr *ifma;
int count;
count = 0;
NET_EPOCH_ENTER(et);
CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link)
if (ifma->ifma_addr->sa_family == AF_LINK)
count++;
NET_EPOCH_EXIT(et);
return (count);
}
u_int
if_foreach_llmaddr(if_t ifp, iflladdr_cb_t cb, void *cb_arg)
{
struct epoch_tracker et;
struct ifmultiaddr *ifma;
u_int count;
MPASS(cb);
count = 0;
NET_EPOCH_ENTER(et);
CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
if (ifma->ifma_addr->sa_family != AF_LINK)
continue;
count += (*cb)(cb_arg, (struct sockaddr_dl *)ifma->ifma_addr,
count);
}
NET_EPOCH_EXIT(et);
return (count);
}
int
if_setsoftc(if_t ifp, void *softc)
{
((struct ifnet *)ifp)->if_softc = softc;
return (0);
}
void *
if_getsoftc(if_t ifp)
{
return ((struct ifnet *)ifp)->if_softc;
}
void
if_setrcvif(struct mbuf *m, if_t ifp)
{
MPASS((m->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0);
m->m_pkthdr.rcvif = (struct ifnet *)ifp;
}
void
if_setvtag(struct mbuf *m, uint16_t tag)
{
m->m_pkthdr.ether_vtag = tag;
}
uint16_t
if_getvtag(struct mbuf *m)
{
return (m->m_pkthdr.ether_vtag);
}
int
if_sendq_empty(if_t ifp)
{
return IFQ_DRV_IS_EMPTY(&((struct ifnet *)ifp)->if_snd);
}
struct ifaddr *
if_getifaddr(if_t ifp)
{
return ((struct ifnet *)ifp)->if_addr;
}
int
if_getamcount(if_t ifp)
{
return ((struct ifnet *)ifp)->if_amcount;
}
int
if_setsendqready(if_t ifp)
{
IFQ_SET_READY(&((struct ifnet *)ifp)->if_snd);
return (0);
}
int
if_setsendqlen(if_t ifp, int tx_desc_count)
{
IFQ_SET_MAXLEN(&((struct ifnet *)ifp)->if_snd, tx_desc_count);
((struct ifnet *)ifp)->if_snd.ifq_drv_maxlen = tx_desc_count;
return (0);
}
int
if_vlantrunkinuse(if_t ifp)
{
return ((struct ifnet *)ifp)->if_vlantrunk != NULL?1:0;
}
int
if_input(if_t ifp, struct mbuf* sendmp)
{
(*((struct ifnet *)ifp)->if_input)((struct ifnet *)ifp, sendmp);
return (0);
}
struct mbuf *
if_dequeue(if_t ifp)
{
struct mbuf *m;
IFQ_DRV_DEQUEUE(&((struct ifnet *)ifp)->if_snd, m);
return (m);
}
int
if_sendq_prepend(if_t ifp, struct mbuf *m)
{
IFQ_DRV_PREPEND(&((struct ifnet *)ifp)->if_snd, m);
return (0);
}
int
if_setifheaderlen(if_t ifp, int len)
{
((struct ifnet *)ifp)->if_hdrlen = len;
return (0);
}
caddr_t
if_getlladdr(if_t ifp)
{
return (IF_LLADDR((struct ifnet *)ifp));
}
void *
if_gethandle(u_char type)
{
return (if_alloc(type));
}
void
if_bpfmtap(if_t ifh, struct mbuf *m)
{
struct ifnet *ifp = (struct ifnet *)ifh;
BPF_MTAP(ifp, m);
}
void
if_etherbpfmtap(if_t ifh, struct mbuf *m)
{
struct ifnet *ifp = (struct ifnet *)ifh;
ETHER_BPF_MTAP(ifp, m);
}
void
if_vlancap(if_t ifh)
{
struct ifnet *ifp = (struct ifnet *)ifh;
VLAN_CAPABILITIES(ifp);
}
int
if_sethwtsomax(if_t ifp, u_int if_hw_tsomax)
{
((struct ifnet *)ifp)->if_hw_tsomax = if_hw_tsomax;
return (0);
}
int
if_sethwtsomaxsegcount(if_t ifp, u_int if_hw_tsomaxsegcount)
{
((struct ifnet *)ifp)->if_hw_tsomaxsegcount = if_hw_tsomaxsegcount;
return (0);
}
int
if_sethwtsomaxsegsize(if_t ifp, u_int if_hw_tsomaxsegsize)
{
((struct ifnet *)ifp)->if_hw_tsomaxsegsize = if_hw_tsomaxsegsize;
return (0);
}
u_int
if_gethwtsomax(if_t ifp)
{
return (((struct ifnet *)ifp)->if_hw_tsomax);
}
u_int
if_gethwtsomaxsegcount(if_t ifp)
{
return (((struct ifnet *)ifp)->if_hw_tsomaxsegcount);
}
u_int
if_gethwtsomaxsegsize(if_t ifp)
{
return (((struct ifnet *)ifp)->if_hw_tsomaxsegsize);
}
void
if_setinitfn(if_t ifp, void (*init_fn)(void *))
{
((struct ifnet *)ifp)->if_init = init_fn;
}
void
if_setioctlfn(if_t ifp, int (*ioctl_fn)(if_t, u_long, caddr_t))
{
((struct ifnet *)ifp)->if_ioctl = (void *)ioctl_fn;
}
void
if_setstartfn(if_t ifp, void (*start_fn)(if_t))
{
((struct ifnet *)ifp)->if_start = (void *)start_fn;
}
void
if_settransmitfn(if_t ifp, if_transmit_fn_t start_fn)
{
((struct ifnet *)ifp)->if_transmit = start_fn;
}
void if_setqflushfn(if_t ifp, if_qflush_fn_t flush_fn)
{
((struct ifnet *)ifp)->if_qflush = flush_fn;
}
void
if_setgetcounterfn(if_t ifp, if_get_counter_t fn)
{
ifp->if_get_counter = fn;
}
#ifdef DDB
static void
if_show_ifnet(struct ifnet *ifp)
{
if (ifp == NULL)
return;
db_printf("%s:\n", ifp->if_xname);
#define IF_DB_PRINTF(f, e) db_printf(" %s = " f "\n", #e, ifp->e);
IF_DB_PRINTF("%s", if_dname);
IF_DB_PRINTF("%d", if_dunit);
IF_DB_PRINTF("%s", if_description);
IF_DB_PRINTF("%u", if_index);
- IF_DB_PRINTF("%d", if_idxgen);
IF_DB_PRINTF("%u", if_refcount);
+ IF_DB_PRINTF("%d", if_index_reserved);
IF_DB_PRINTF("%p", if_softc);
IF_DB_PRINTF("%p", if_l2com);
IF_DB_PRINTF("%p", if_llsoftc);
IF_DB_PRINTF("%d", if_amcount);
IF_DB_PRINTF("%p", if_addr);
IF_DB_PRINTF("%p", if_broadcastaddr);
IF_DB_PRINTF("%p", if_afdata);
IF_DB_PRINTF("%d", if_afdata_initialized);
IF_DB_PRINTF("%u", if_fib);
IF_DB_PRINTF("%p", if_vnet);
IF_DB_PRINTF("%p", if_home_vnet);
IF_DB_PRINTF("%p", if_vlantrunk);
IF_DB_PRINTF("%p", if_bpf);
IF_DB_PRINTF("%u", if_pcount);
IF_DB_PRINTF("%p", if_bridge);
IF_DB_PRINTF("%p", if_lagg);
IF_DB_PRINTF("%p", if_pf_kif);
IF_DB_PRINTF("%p", if_carp);
IF_DB_PRINTF("%p", if_label);
IF_DB_PRINTF("%p", if_netmap);
IF_DB_PRINTF("0x%08x", if_flags);
IF_DB_PRINTF("0x%08x", if_drv_flags);
IF_DB_PRINTF("0x%08x", if_capabilities);
IF_DB_PRINTF("0x%08x", if_capenable);
IF_DB_PRINTF("%p", if_snd.ifq_head);
IF_DB_PRINTF("%p", if_snd.ifq_tail);
IF_DB_PRINTF("%d", if_snd.ifq_len);
IF_DB_PRINTF("%d", if_snd.ifq_maxlen);
IF_DB_PRINTF("%p", if_snd.ifq_drv_head);
IF_DB_PRINTF("%p", if_snd.ifq_drv_tail);
IF_DB_PRINTF("%d", if_snd.ifq_drv_len);
IF_DB_PRINTF("%d", if_snd.ifq_drv_maxlen);
IF_DB_PRINTF("%d", if_snd.altq_type);
IF_DB_PRINTF("%x", if_snd.altq_flags);
#undef IF_DB_PRINTF
}
DB_SHOW_COMMAND(ifnet, db_show_ifnet)
{
if (!have_addr) {
db_printf("usage: show ifnet <struct ifnet *>\n");
return;
}
if_show_ifnet((struct ifnet *)addr);
}
DB_SHOW_ALL_COMMAND(ifnets, db_show_all_ifnets)
{
struct ifnet *ifp;
u_short idx;
for (idx = 1; idx <= if_index; idx++) {
- ifp = ifindex_table[idx].ife_ifnet;
+ ifp = ifindex_table[idx];
if (ifp == NULL)
continue;
db_printf( "%20s ifp=%p\n", ifp->if_xname, ifp);
if (db_pager_quit)
break;
}
}
#endif /* DDB */
diff --git a/sys/net/if_var.h b/sys/net/if_var.h
index e054c613e9e6..d741d8abee2e 100644
--- a/sys/net/if_var.h
+++ b/sys/net/if_var.h
@@ -1,823 +1,816 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1982, 1986, 1989, 1993
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* From: @(#)if.h 8.1 (Berkeley) 6/10/93
* $FreeBSD$
*/
#ifndef _NET_IF_VAR_H_
#define _NET_IF_VAR_H_
/*
* Structures defining a network interface, providing a packet
* transport mechanism (ala level 0 of the PUP protocols).
*
* Each interface accepts output datagrams of a specified maximum
* length, and provides higher level routines with input datagrams
* received from its medium.
*
* Output occurs when the routine if_output is called, with three parameters:
* (*ifp->if_output)(ifp, m, dst, ro)
* Here m is the mbuf chain to be sent and dst is the destination address.
* The output routine encapsulates the supplied datagram if necessary,
* and then transmits it on its medium.
*
* On input, each interface unwraps the data received by it, and either
* places it on the input queue of an internetwork datagram routine
* and posts the associated software interrupt, or passes the datagram to a raw
* packet input routine.
*
* Routines exist for locating interfaces by their addresses
* or for locating an interface on a certain network, as well as more general
* routing and gateway routines maintaining information used to locate
* interfaces. These routines live in the files if.c and route.c
*/
struct rtentry; /* ifa_rtrequest */
struct socket;
struct carp_if;
struct carp_softc;
struct ifvlantrunk;
struct route; /* if_output */
struct vnet;
struct ifmedia;
struct netmap_adapter;
struct debugnet_methods;
#ifdef _KERNEL
#include <sys/_eventhandler.h>
#include <sys/mbuf.h> /* ifqueue only? */
#include <sys/buf_ring.h>
#include <net/vnet.h>
#endif /* _KERNEL */
#include <sys/ck.h>
#include <sys/counter.h>
#include <sys/epoch.h>
#include <sys/lock.h> /* XXX */
#include <sys/mutex.h> /* struct ifqueue */
#include <sys/rwlock.h> /* XXX */
#include <sys/sx.h> /* XXX */
#include <sys/_task.h> /* if_link_task */
#define IF_DUNIT_NONE -1
#include <net/altq/if_altq.h>
CK_STAILQ_HEAD(ifnethead, ifnet); /* we use TAILQs so that the order of */
CK_STAILQ_HEAD(ifaddrhead, ifaddr); /* instantiation is preserved in the list */
CK_STAILQ_HEAD(ifmultihead, ifmultiaddr);
CK_STAILQ_HEAD(ifgrouphead, ifg_group);
#ifdef _KERNEL
VNET_DECLARE(struct pfil_head *, link_pfil_head);
#define V_link_pfil_head VNET(link_pfil_head)
#define PFIL_ETHER_NAME "ethernet"
#define HHOOK_IPSEC_INET 0
#define HHOOK_IPSEC_INET6 1
#define HHOOK_IPSEC_COUNT 2
VNET_DECLARE(struct hhook_head *, ipsec_hhh_in[HHOOK_IPSEC_COUNT]);
VNET_DECLARE(struct hhook_head *, ipsec_hhh_out[HHOOK_IPSEC_COUNT]);
#define V_ipsec_hhh_in VNET(ipsec_hhh_in)
#define V_ipsec_hhh_out VNET(ipsec_hhh_out)
#endif /* _KERNEL */
typedef enum {
IFCOUNTER_IPACKETS = 0,
IFCOUNTER_IERRORS,
IFCOUNTER_OPACKETS,
IFCOUNTER_OERRORS,
IFCOUNTER_COLLISIONS,
IFCOUNTER_IBYTES,
IFCOUNTER_OBYTES,
IFCOUNTER_IMCASTS,
IFCOUNTER_OMCASTS,
IFCOUNTER_IQDROPS,
IFCOUNTER_OQDROPS,
IFCOUNTER_NOPROTO,
IFCOUNTERS /* Array size. */
} ift_counter;
typedef struct ifnet * if_t;
typedef void (*if_start_fn_t)(if_t);
typedef int (*if_ioctl_fn_t)(if_t, u_long, caddr_t);
typedef void (*if_init_fn_t)(void *);
typedef void (*if_qflush_fn_t)(if_t);
typedef int (*if_transmit_fn_t)(if_t, struct mbuf *);
typedef uint64_t (*if_get_counter_t)(if_t, ift_counter);
struct ifnet_hw_tsomax {
u_int tsomaxbytes; /* TSO total burst length limit in bytes */
u_int tsomaxsegcount; /* TSO maximum segment count */
u_int tsomaxsegsize; /* TSO maximum segment size in bytes */
};
/* Interface encap request types */
typedef enum {
IFENCAP_LL = 1 /* pre-calculate link-layer header */
} ife_type;
/*
* The structure below allows to request various pre-calculated L2/L3 headers
* for different media. Requests varies by type (rtype field).
*
* IFENCAP_LL type: pre-calculates link header based on address family
* and destination lladdr.
*
* Input data fields:
* buf: pointer to destination buffer
* bufsize: buffer size
* flags: IFENCAP_FLAG_BROADCAST if destination is broadcast
* family: address family defined by AF_ constant.
* lladdr: pointer to link-layer address
* lladdr_len: length of link-layer address
* hdata: pointer to L3 header (optional, used for ARP requests).
* Output data fields:
* buf: encap data is stored here
* bufsize: resulting encap length is stored here
* lladdr_off: offset of link-layer address from encap hdr start
* hdata: L3 header may be altered if necessary
*/
struct if_encap_req {
u_char *buf; /* Destination buffer (w) */
size_t bufsize; /* size of provided buffer (r) */
ife_type rtype; /* request type (r) */
uint32_t flags; /* Request flags (r) */
int family; /* Address family AF_* (r) */
int lladdr_off; /* offset from header start (w) */
int lladdr_len; /* lladdr length (r) */
char *lladdr; /* link-level address pointer (r) */
char *hdata; /* Upper layer header data (rw) */
};
#define IFENCAP_FLAG_BROADCAST 0x02 /* Destination is broadcast */
/*
* Network interface send tag support. The storage of "struct
* m_snd_tag" comes from the network driver and it is free to allocate
* as much additional space as it wants for its own use.
*/
struct ktls_session;
struct m_snd_tag;
#define IF_SND_TAG_TYPE_RATE_LIMIT 0
#define IF_SND_TAG_TYPE_UNLIMITED 1
#define IF_SND_TAG_TYPE_TLS 2
#define IF_SND_TAG_TYPE_TLS_RATE_LIMIT 3
#define IF_SND_TAG_TYPE_TLS_RX 4
#define IF_SND_TAG_TYPE_MAX 5
struct if_snd_tag_alloc_header {
uint32_t type; /* send tag type, see IF_SND_TAG_XXX */
uint32_t flowid; /* mbuf hash value */
uint32_t flowtype; /* mbuf hash type */
uint8_t numa_domain; /* numa domain of associated inp */
};
struct if_snd_tag_alloc_rate_limit {
struct if_snd_tag_alloc_header hdr;
uint64_t max_rate; /* in bytes/s */
uint32_t flags; /* M_NOWAIT or M_WAITOK */
uint32_t reserved; /* alignment */
};
struct if_snd_tag_alloc_tls {
struct if_snd_tag_alloc_header hdr;
struct inpcb *inp;
const struct ktls_session *tls;
};
struct if_snd_tag_alloc_tls_rx {
struct if_snd_tag_alloc_header hdr;
struct inpcb *inp;
const struct ktls_session *tls;
uint16_t vlan_id; /* valid if non-zero */
};
struct if_snd_tag_alloc_tls_rate_limit {
struct if_snd_tag_alloc_header hdr;
struct inpcb *inp;
const struct ktls_session *tls;
uint64_t max_rate; /* in bytes/s */
};
struct if_snd_tag_rate_limit_params {
uint64_t max_rate; /* in bytes/s */
uint32_t queue_level; /* 0 (empty) .. 65535 (full) */
#define IF_SND_QUEUE_LEVEL_MIN 0
#define IF_SND_QUEUE_LEVEL_MAX 65535
uint32_t flags; /* M_NOWAIT or M_WAITOK */
};
struct if_snd_tag_modify_tls_rx {
/* TCP sequence number of TLS header in host endian format */
uint32_t tls_hdr_tcp_sn;
/*
* TLS record length, including all headers, data and trailers.
* If the tls_rec_length is zero, it means HW encryption resumed.
*/
uint32_t tls_rec_length;
/* TLS sequence number in host endian format */
uint64_t tls_seq_number;
};
union if_snd_tag_alloc_params {
struct if_snd_tag_alloc_header hdr;
struct if_snd_tag_alloc_rate_limit rate_limit;
struct if_snd_tag_alloc_rate_limit unlimited;
struct if_snd_tag_alloc_tls tls;
struct if_snd_tag_alloc_tls_rx tls_rx;
struct if_snd_tag_alloc_tls_rate_limit tls_rate_limit;
};
union if_snd_tag_modify_params {
struct if_snd_tag_rate_limit_params rate_limit;
struct if_snd_tag_rate_limit_params unlimited;
struct if_snd_tag_rate_limit_params tls_rate_limit;
struct if_snd_tag_modify_tls_rx tls_rx;
};
union if_snd_tag_query_params {
struct if_snd_tag_rate_limit_params rate_limit;
struct if_snd_tag_rate_limit_params unlimited;
struct if_snd_tag_rate_limit_params tls_rate_limit;
};
typedef int (if_snd_tag_alloc_t)(struct ifnet *, union if_snd_tag_alloc_params *,
struct m_snd_tag **);
typedef int (if_snd_tag_modify_t)(struct m_snd_tag *, union if_snd_tag_modify_params *);
typedef int (if_snd_tag_query_t)(struct m_snd_tag *, union if_snd_tag_query_params *);
typedef void (if_snd_tag_free_t)(struct m_snd_tag *);
typedef struct m_snd_tag *(if_next_send_tag_t)(struct m_snd_tag *);
struct if_snd_tag_sw {
if_snd_tag_modify_t *snd_tag_modify;
if_snd_tag_query_t *snd_tag_query;
if_snd_tag_free_t *snd_tag_free;
if_next_send_tag_t *next_snd_tag;
u_int type; /* One of IF_SND_TAG_TYPE_*. */
};
/* Query return flags */
#define RT_NOSUPPORT 0x00000000 /* Not supported */
#define RT_IS_INDIRECT 0x00000001 /*
* Interface like a lagg, select
* the actual interface for
* capabilities.
*/
#define RT_IS_SELECTABLE 0x00000002 /*
* No rate table, you select
* rates and the first
* number_of_rates are created.
*/
#define RT_IS_FIXED_TABLE 0x00000004 /* A fixed table is attached */
#define RT_IS_UNUSABLE 0x00000008 /* It is not usable for this */
#define RT_IS_SETUP_REQ 0x00000010 /* The interface setup must be called before use */
struct if_ratelimit_query_results {
const uint64_t *rate_table; /* Pointer to table if present */
uint32_t flags; /* Flags indicating results */
uint32_t max_flows; /* Max flows using, 0=unlimited */
uint32_t number_of_rates; /* How many unique rates can be created */
uint32_t min_segment_burst; /* The amount the adapter bursts at each send */
};
typedef void (if_ratelimit_query_t)(struct ifnet *,
struct if_ratelimit_query_results *);
typedef int (if_ratelimit_setup_t)(struct ifnet *, uint64_t, uint32_t);
/*
* Structure defining a network interface.
*/
struct ifnet {
/* General book keeping of interface lists. */
CK_STAILQ_ENTRY(ifnet) if_link; /* all struct ifnets are chained (CK_) */
LIST_ENTRY(ifnet) if_clones; /* interfaces of a cloner */
CK_STAILQ_HEAD(, ifg_list) if_groups; /* linked list of groups per if (CK_) */
/* protected by if_addr_lock */
u_char if_alloctype; /* if_type at time of allocation */
uint8_t if_numa_domain; /* NUMA domain of device */
/* Driver and protocol specific information that remains stable. */
void *if_softc; /* pointer to driver state */
void *if_llsoftc; /* link layer softc */
void *if_l2com; /* pointer to protocol bits */
const char *if_dname; /* driver name */
int if_dunit; /* unit or IF_DUNIT_NONE */
u_short if_index; /* numeric abbreviation for this if */
- u_short if_idxgen; /* ... and its generation count */
+ short if_index_reserved; /* spare space to grow if_index */
char if_xname[IFNAMSIZ]; /* external name (name + unit) */
char *if_description; /* interface description */
/* Variable fields that are touched by the stack and drivers. */
int if_flags; /* up/down, broadcast, etc. */
int if_drv_flags; /* driver-managed status flags */
int if_capabilities; /* interface features & capabilities */
int if_capenable; /* enabled features & capabilities */
void *if_linkmib; /* link-type-specific MIB data */
size_t if_linkmiblen; /* length of above data */
u_int if_refcount; /* reference count */
/* These fields are shared with struct if_data. */
uint8_t if_type; /* ethernet, tokenring, etc */
uint8_t if_addrlen; /* media address length */
uint8_t if_hdrlen; /* media header length */
uint8_t if_link_state; /* current link state */
uint32_t if_mtu; /* maximum transmission unit */
uint32_t if_metric; /* routing metric (external only) */
uint64_t if_baudrate; /* linespeed */
uint64_t if_hwassist; /* HW offload capabilities, see IFCAP */
time_t if_epoch; /* uptime at attach or stat reset */
struct timeval if_lastchange; /* time of last administrative change */
struct ifaltq if_snd; /* output queue (includes altq) */
struct task if_linktask; /* task for link change events */
struct task if_addmultitask; /* task for SIOCADDMULTI */
/* Addresses of different protocol families assigned to this if. */
struct mtx if_addr_lock; /* lock to protect address lists */
/*
* if_addrhead is the list of all addresses associated to
* an interface.
* Some code in the kernel assumes that first element
* of the list has type AF_LINK, and contains sockaddr_dl
* addresses which store the link-level address and the name
* of the interface.
* However, access to the AF_LINK address through this
* field is deprecated. Use if_addr instead.
*/
struct ifaddrhead if_addrhead; /* linked list of addresses per if */
struct ifmultihead if_multiaddrs; /* multicast addresses configured */
int if_amcount; /* number of all-multicast requests */
struct ifaddr *if_addr; /* pointer to link-level address */
void *if_hw_addr; /* hardware link-level address */
const u_int8_t *if_broadcastaddr; /* linklevel broadcast bytestring */
struct mtx if_afdata_lock;
void *if_afdata[AF_MAX];
int if_afdata_initialized;
/* Additional features hung off the interface. */
u_int if_fib; /* interface FIB */
struct vnet *if_vnet; /* pointer to network stack instance */
struct vnet *if_home_vnet; /* where this ifnet originates from */
struct ifvlantrunk *if_vlantrunk; /* pointer to 802.1q data */
struct bpf_if *if_bpf; /* packet filter structure */
int if_pcount; /* number of promiscuous listeners */
void *if_bridge; /* bridge glue */
void *if_lagg; /* lagg glue */
void *if_pf_kif; /* pf glue */
struct carp_if *if_carp; /* carp interface structure */
struct label *if_label; /* interface MAC label */
struct netmap_adapter *if_netmap; /* netmap(4) softc */
/* Various procedures of the layer2 encapsulation and drivers. */
int (*if_output) /* output routine (enqueue) */
(struct ifnet *, struct mbuf *, const struct sockaddr *,
struct route *);
void (*if_input) /* input routine (from h/w driver) */
(struct ifnet *, struct mbuf *);
struct mbuf *(*if_bridge_input)(struct ifnet *, struct mbuf *);
int (*if_bridge_output)(struct ifnet *, struct mbuf *, struct sockaddr *,
struct rtentry *);
void (*if_bridge_linkstate)(struct ifnet *ifp);
if_start_fn_t if_start; /* initiate output routine */
if_ioctl_fn_t if_ioctl; /* ioctl routine */
if_init_fn_t if_init; /* Init routine */
int (*if_resolvemulti) /* validate/resolve multicast */
(struct ifnet *, struct sockaddr **, struct sockaddr *);
if_qflush_fn_t if_qflush; /* flush any queue */
if_transmit_fn_t if_transmit; /* initiate output routine */
void (*if_reassign) /* reassign to vnet routine */
(struct ifnet *, struct vnet *, char *);
if_get_counter_t if_get_counter; /* get counter values */
int (*if_requestencap) /* make link header from request */
(struct ifnet *, struct if_encap_req *);
/* Statistics. */
counter_u64_t if_counters[IFCOUNTERS];
/* Stuff that's only temporary and doesn't belong here. */
/*
* Network adapter TSO limits:
* ===========================
*
* If the "if_hw_tsomax" field is zero the maximum segment
* length limit does not apply. If the "if_hw_tsomaxsegcount"
* or the "if_hw_tsomaxsegsize" field is zero the TSO segment
* count limit does not apply. If all three fields are zero,
* there is no TSO limit.
*
* NOTE: The TSO limits should reflect the values used in the
* BUSDMA tag a network adapter is using to load a mbuf chain
* for transmission. The TCP/IP network stack will subtract
* space for all linklevel and protocol level headers and
* ensure that the full mbuf chain passed to the network
* adapter fits within the given limits.
*/
u_int if_hw_tsomax; /* TSO maximum size in bytes */
u_int if_hw_tsomaxsegcount; /* TSO maximum segment count */
u_int if_hw_tsomaxsegsize; /* TSO maximum segment size in bytes */
/*
* Network adapter send tag support:
*/
if_snd_tag_alloc_t *if_snd_tag_alloc;
/* Ratelimit (packet pacing) */
if_ratelimit_query_t *if_ratelimit_query;
if_ratelimit_setup_t *if_ratelimit_setup;
/* Ethernet PCP */
uint8_t if_pcp;
/*
* Debugnet (Netdump) hooks to be called while in db/panic.
*/
struct debugnet_methods *if_debugnet_methods;
struct epoch_context if_epoch_ctx;
/*
* Spare fields to be added before branching a stable branch, so
* that structure can be enhanced without changing the kernel
* binary interface.
*/
int if_ispare[4]; /* general use */
};
/* for compatibility with other BSDs */
#define if_name(ifp) ((ifp)->if_xname)
#define IF_NODOM 255
/*
* Locks for address lists on the network interface.
*/
#define IF_ADDR_LOCK_INIT(if) mtx_init(&(if)->if_addr_lock, "if_addr_lock", NULL, MTX_DEF)
#define IF_ADDR_LOCK_DESTROY(if) mtx_destroy(&(if)->if_addr_lock)
#define IF_ADDR_WLOCK(if) mtx_lock(&(if)->if_addr_lock)
#define IF_ADDR_WUNLOCK(if) mtx_unlock(&(if)->if_addr_lock)
#define IF_ADDR_LOCK_ASSERT(if) MPASS(in_epoch(net_epoch_preempt) || mtx_owned(&(if)->if_addr_lock))
#define IF_ADDR_WLOCK_ASSERT(if) mtx_assert(&(if)->if_addr_lock, MA_OWNED)
#ifdef _KERNEL
/* interface link layer address change event */
typedef void (*iflladdr_event_handler_t)(void *, struct ifnet *);
EVENTHANDLER_DECLARE(iflladdr_event, iflladdr_event_handler_t);
/* interface address change event */
typedef void (*ifaddr_event_handler_t)(void *, struct ifnet *);
EVENTHANDLER_DECLARE(ifaddr_event, ifaddr_event_handler_t);
typedef void (*ifaddr_event_ext_handler_t)(void *, struct ifnet *,
struct ifaddr *, int);
EVENTHANDLER_DECLARE(ifaddr_event_ext, ifaddr_event_ext_handler_t);
#define IFADDR_EVENT_ADD 0
#define IFADDR_EVENT_DEL 1
/* new interface arrival event */
typedef void (*ifnet_arrival_event_handler_t)(void *, struct ifnet *);
EVENTHANDLER_DECLARE(ifnet_arrival_event, ifnet_arrival_event_handler_t);
/* interface departure event */
typedef void (*ifnet_departure_event_handler_t)(void *, struct ifnet *);
EVENTHANDLER_DECLARE(ifnet_departure_event, ifnet_departure_event_handler_t);
/* Interface link state change event */
typedef void (*ifnet_link_event_handler_t)(void *, struct ifnet *, int);
EVENTHANDLER_DECLARE(ifnet_link_event, ifnet_link_event_handler_t);
/* Interface up/down event */
#define IFNET_EVENT_UP 0
#define IFNET_EVENT_DOWN 1
#define IFNET_EVENT_PCP 2 /* priority code point, PCP */
typedef void (*ifnet_event_fn)(void *, struct ifnet *ifp, int event);
EVENTHANDLER_DECLARE(ifnet_event, ifnet_event_fn);
/*
* interface groups
*/
struct ifg_group {
char ifg_group[IFNAMSIZ];
u_int ifg_refcnt;
void *ifg_pf_kif;
CK_STAILQ_HEAD(, ifg_member) ifg_members; /* (CK_) */
CK_STAILQ_ENTRY(ifg_group) ifg_next; /* (CK_) */
};
struct ifg_member {
CK_STAILQ_ENTRY(ifg_member) ifgm_next; /* (CK_) */
struct ifnet *ifgm_ifp;
};
struct ifg_list {
struct ifg_group *ifgl_group;
CK_STAILQ_ENTRY(ifg_list) ifgl_next; /* (CK_) */
};
#ifdef _SYS_EVENTHANDLER_H_
/* group attach event */
typedef void (*group_attach_event_handler_t)(void *, struct ifg_group *);
EVENTHANDLER_DECLARE(group_attach_event, group_attach_event_handler_t);
/* group detach event */
typedef void (*group_detach_event_handler_t)(void *, struct ifg_group *);
EVENTHANDLER_DECLARE(group_detach_event, group_detach_event_handler_t);
/* group change event */
typedef void (*group_change_event_handler_t)(void *, const char *);
EVENTHANDLER_DECLARE(group_change_event, group_change_event_handler_t);
#endif /* _SYS_EVENTHANDLER_H_ */
#define IF_AFDATA_LOCK_INIT(ifp) \
mtx_init(&(ifp)->if_afdata_lock, "if_afdata", NULL, MTX_DEF)
#define IF_AFDATA_WLOCK(ifp) mtx_lock(&(ifp)->if_afdata_lock)
#define IF_AFDATA_WUNLOCK(ifp) mtx_unlock(&(ifp)->if_afdata_lock)
#define IF_AFDATA_LOCK(ifp) IF_AFDATA_WLOCK(ifp)
#define IF_AFDATA_UNLOCK(ifp) IF_AFDATA_WUNLOCK(ifp)
#define IF_AFDATA_TRYLOCK(ifp) mtx_trylock(&(ifp)->if_afdata_lock)
#define IF_AFDATA_DESTROY(ifp) mtx_destroy(&(ifp)->if_afdata_lock)
#define IF_AFDATA_LOCK_ASSERT(ifp) MPASS(in_epoch(net_epoch_preempt) || mtx_owned(&(ifp)->if_afdata_lock))
#define IF_AFDATA_WLOCK_ASSERT(ifp) mtx_assert(&(ifp)->if_afdata_lock, MA_OWNED)
#define IF_AFDATA_UNLOCK_ASSERT(ifp) mtx_assert(&(ifp)->if_afdata_lock, MA_NOTOWNED)
/*
* 72 was chosen below because it is the size of a TCP/IP
* header (40) + the minimum mss (32).
*/
#define IF_MINMTU 72
#define IF_MAXMTU 65535
#define TOEDEV(ifp) ((ifp)->if_llsoftc)
/*
* The ifaddr structure contains information about one address
* of an interface. They are maintained by the different address families,
* are allocated and attached when an address is set, and are linked
* together so all addresses for an interface can be located.
*
* NOTE: a 'struct ifaddr' is always at the beginning of a larger
* chunk of malloc'ed memory, where we store the three addresses
* (ifa_addr, ifa_dstaddr and ifa_netmask) referenced here.
*/
struct ifaddr {
struct sockaddr *ifa_addr; /* address of interface */
struct sockaddr *ifa_dstaddr; /* other end of p-to-p link */
#define ifa_broadaddr ifa_dstaddr /* broadcast address interface */
struct sockaddr *ifa_netmask; /* used to determine subnet */
struct ifnet *ifa_ifp; /* back-pointer to interface */
struct carp_softc *ifa_carp; /* pointer to CARP data */
CK_STAILQ_ENTRY(ifaddr) ifa_link; /* queue macro glue */
u_short ifa_flags; /* mostly rt_flags for cloning */
#define IFA_ROUTE RTF_UP /* route installed */
#define IFA_RTSELF RTF_HOST /* loopback route to self installed */
u_int ifa_refcnt; /* references to this structure */
counter_u64_t ifa_ipackets;
counter_u64_t ifa_opackets;
counter_u64_t ifa_ibytes;
counter_u64_t ifa_obytes;
struct epoch_context ifa_epoch_ctx;
};
struct ifaddr * ifa_alloc(size_t size, int flags);
void ifa_free(struct ifaddr *ifa);
void ifa_ref(struct ifaddr *ifa);
int __result_use_check ifa_try_ref(struct ifaddr *ifa);
/*
* Multicast address structure. This is analogous to the ifaddr
* structure except that it keeps track of multicast addresses.
*/
#define IFMA_F_ENQUEUED 0x1
struct ifmultiaddr {
CK_STAILQ_ENTRY(ifmultiaddr) ifma_link; /* queue macro glue */
struct sockaddr *ifma_addr; /* address this membership is for */
struct sockaddr *ifma_lladdr; /* link-layer translation, if any */
struct ifnet *ifma_ifp; /* back-pointer to interface */
u_int ifma_refcount; /* reference count */
int ifma_flags;
void *ifma_protospec; /* protocol-specific state, if any */
struct ifmultiaddr *ifma_llifma; /* pointer to ifma for ifma_lladdr */
struct epoch_context ifma_epoch_ctx;
};
extern struct sx ifnet_sxlock;
#define IFNET_WLOCK() sx_xlock(&ifnet_sxlock)
#define IFNET_WUNLOCK() sx_xunlock(&ifnet_sxlock)
#define IFNET_RLOCK_ASSERT() sx_assert(&ifnet_sxlock, SA_SLOCKED)
#define IFNET_WLOCK_ASSERT() sx_assert(&ifnet_sxlock, SA_XLOCKED)
#define IFNET_RLOCK() sx_slock(&ifnet_sxlock)
#define IFNET_RUNLOCK() sx_sunlock(&ifnet_sxlock)
/*
* Look up an ifnet given its index. The returned value protected from
* being freed by the network epoch. The _ref variant also acquires a
* reference that must be freed using if_rele().
*/
struct ifnet *ifnet_byindex(u_int);
struct ifnet *ifnet_byindex_ref(u_int);
-/*
- * ifnet_byindexgen() looks up ifnet by index and generation count,
- * attempting to restore a weak pointer that had been stored across
- * the epoch.
- */
-struct ifnet *ifnet_byindexgen(uint16_t idx, uint16_t gen);
-
VNET_DECLARE(struct ifnethead, ifnet);
VNET_DECLARE(struct ifgrouphead, ifg_head);
VNET_DECLARE(struct ifnet *, loif); /* first loopback interface */
#define V_ifnet VNET(ifnet)
#define V_ifg_head VNET(ifg_head)
#define V_loif VNET(loif)
#ifdef MCAST_VERBOSE
#define MCDPRINTF printf
#else
#define MCDPRINTF(...)
#endif
int if_addgroup(struct ifnet *, const char *);
int if_delgroup(struct ifnet *, const char *);
int if_addmulti(struct ifnet *, struct sockaddr *, struct ifmultiaddr **);
int if_allmulti(struct ifnet *, int);
struct ifnet* if_alloc(u_char);
struct ifnet* if_alloc_dev(u_char, device_t dev);
void if_attach(struct ifnet *);
void if_dead(struct ifnet *);
int if_delmulti(struct ifnet *, struct sockaddr *);
void if_delmulti_ifma(struct ifmultiaddr *);
void if_delmulti_ifma_flags(struct ifmultiaddr *, int flags);
void if_detach(struct ifnet *);
void if_purgeaddrs(struct ifnet *);
void if_delallmulti(struct ifnet *);
void if_down(struct ifnet *);
struct ifmultiaddr *
if_findmulti(struct ifnet *, const struct sockaddr *);
void if_freemulti(struct ifmultiaddr *ifma);
void if_free(struct ifnet *);
void if_initname(struct ifnet *, const char *, int);
void if_link_state_change(struct ifnet *, int);
int if_printf(struct ifnet *, const char *, ...) __printflike(2, 3);
int if_log(struct ifnet *, int, const char *, ...) __printflike(3, 4);
void if_ref(struct ifnet *);
void if_rele(struct ifnet *);
bool __result_use_check if_try_ref(struct ifnet *);
int if_setlladdr(struct ifnet *, const u_char *, int);
int if_tunnel_check_nesting(struct ifnet *, struct mbuf *, uint32_t, int);
void if_up(struct ifnet *);
int ifioctl(struct socket *, u_long, caddr_t, struct thread *);
int ifpromisc(struct ifnet *, int);
struct ifnet *ifunit(const char *);
struct ifnet *ifunit_ref(const char *);
int ifa_add_loopback_route(struct ifaddr *, struct sockaddr *);
int ifa_del_loopback_route(struct ifaddr *, struct sockaddr *);
int ifa_switch_loopback_route(struct ifaddr *, struct sockaddr *);
struct ifaddr *ifa_ifwithaddr(const struct sockaddr *);
int ifa_ifwithaddr_check(const struct sockaddr *);
struct ifaddr *ifa_ifwithbroadaddr(const struct sockaddr *, int);
struct ifaddr *ifa_ifwithdstaddr(const struct sockaddr *, int);
struct ifaddr *ifa_ifwithnet(const struct sockaddr *, int, int);
struct ifaddr *ifa_ifwithroute(int, const struct sockaddr *,
const struct sockaddr *, u_int);
struct ifaddr *ifaof_ifpforaddr(const struct sockaddr *, struct ifnet *);
int ifa_preferred(struct ifaddr *, struct ifaddr *);
int if_simloop(struct ifnet *ifp, struct mbuf *m, int af, int hlen);
typedef void *if_com_alloc_t(u_char type, struct ifnet *ifp);
typedef void if_com_free_t(void *com, u_char type);
void if_register_com_alloc(u_char type, if_com_alloc_t *a, if_com_free_t *f);
void if_deregister_com_alloc(u_char type);
void if_data_copy(struct ifnet *, struct if_data *);
uint64_t if_get_counter_default(struct ifnet *, ift_counter);
void if_inc_counter(struct ifnet *, ift_counter, int64_t);
#define IF_LLADDR(ifp) \
LLADDR((struct sockaddr_dl *)((ifp)->if_addr->ifa_addr))
uint64_t if_setbaudrate(if_t ifp, uint64_t baudrate);
uint64_t if_getbaudrate(if_t ifp);
int if_setcapabilities(if_t ifp, int capabilities);
int if_setcapabilitiesbit(if_t ifp, int setbit, int clearbit);
int if_getcapabilities(if_t ifp);
int if_togglecapenable(if_t ifp, int togglecap);
int if_setcapenable(if_t ifp, int capenable);
int if_setcapenablebit(if_t ifp, int setcap, int clearcap);
int if_getcapenable(if_t ifp);
const char *if_getdname(if_t ifp);
int if_setdev(if_t ifp, void *dev);
int if_setdrvflagbits(if_t ifp, int if_setflags, int clear_flags);
int if_getdrvflags(if_t ifp);
int if_setdrvflags(if_t ifp, int flags);
int if_clearhwassist(if_t ifp);
int if_sethwassistbits(if_t ifp, int toset, int toclear);
int if_sethwassist(if_t ifp, int hwassist_bit);
int if_gethwassist(if_t ifp);
int if_setsoftc(if_t ifp, void *softc);
void *if_getsoftc(if_t ifp);
int if_setflags(if_t ifp, int flags);
int if_gethwaddr(if_t ifp, struct ifreq *);
int if_setmtu(if_t ifp, int mtu);
int if_getmtu(if_t ifp);
int if_getmtu_family(if_t ifp, int family);
int if_setflagbits(if_t ifp, int set, int clear);
int if_getflags(if_t ifp);
int if_sendq_empty(if_t ifp);
int if_setsendqready(if_t ifp);
int if_setsendqlen(if_t ifp, int tx_desc_count);
int if_sethwtsomax(if_t ifp, u_int if_hw_tsomax);
int if_sethwtsomaxsegcount(if_t ifp, u_int if_hw_tsomaxsegcount);
int if_sethwtsomaxsegsize(if_t ifp, u_int if_hw_tsomaxsegsize);
u_int if_gethwtsomax(if_t ifp);
u_int if_gethwtsomaxsegcount(if_t ifp);
u_int if_gethwtsomaxsegsize(if_t ifp);
int if_input(if_t ifp, struct mbuf* sendmp);
int if_sendq_prepend(if_t ifp, struct mbuf *m);
struct mbuf *if_dequeue(if_t ifp);
int if_setifheaderlen(if_t ifp, int len);
void if_setrcvif(struct mbuf *m, if_t ifp);
void if_setvtag(struct mbuf *m, u_int16_t tag);
u_int16_t if_getvtag(struct mbuf *m);
int if_vlantrunkinuse(if_t ifp);
caddr_t if_getlladdr(if_t ifp);
void *if_gethandle(u_char);
void if_bpfmtap(if_t ifp, struct mbuf *m);
void if_etherbpfmtap(if_t ifp, struct mbuf *m);
void if_vlancap(if_t ifp);
/*
* Traversing through interface address lists.
*/
struct sockaddr_dl;
typedef u_int iflladdr_cb_t(void *, struct sockaddr_dl *, u_int);
u_int if_foreach_lladdr(if_t, iflladdr_cb_t, void *);
u_int if_foreach_llmaddr(if_t, iflladdr_cb_t, void *);
u_int if_lladdr_count(if_t);
u_int if_llmaddr_count(if_t);
int if_getamcount(if_t ifp);
struct ifaddr * if_getifaddr(if_t ifp);
/* Functions */
void if_setinitfn(if_t ifp, void (*)(void *));
void if_setioctlfn(if_t ifp, int (*)(if_t, u_long, caddr_t));
void if_setstartfn(if_t ifp, void (*)(if_t));
void if_settransmitfn(if_t ifp, if_transmit_fn_t);
void if_setqflushfn(if_t ifp, if_qflush_fn_t);
void if_setgetcounterfn(if_t ifp, if_get_counter_t);
/* TSO */
void if_hw_tsomax_common(if_t ifp, struct ifnet_hw_tsomax *);
int if_hw_tsomax_update(if_t ifp, struct ifnet_hw_tsomax *);
/* accessors for struct ifreq */
void *ifr_data_get_ptr(void *ifrp);
void *ifr_buffer_get_buffer(void *data);
size_t ifr_buffer_get_length(void *data);
int ifhwioctl(u_long, struct ifnet *, caddr_t, struct thread *);
#ifdef DEVICE_POLLING
enum poll_cmd { POLL_ONLY, POLL_AND_CHECK_STATUS };
typedef int poll_handler_t(if_t ifp, enum poll_cmd cmd, int count);
int ether_poll_register(poll_handler_t *h, if_t ifp);
int ether_poll_deregister(if_t ifp);
#endif /* DEVICE_POLLING */
#endif /* _KERNEL */
#include <net/ifq.h> /* XXXAO: temporary unconditional include */
#endif /* !_NET_IF_VAR_H_ */
diff --git a/sys/sys/mbuf.h b/sys/sys/mbuf.h
index 78b175ebe2c0..8ca23b60269d 100644
--- a/sys/sys/mbuf.h
+++ b/sys/sys/mbuf.h
@@ -1,1696 +1,1690 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1982, 1986, 1988, 1993
* The Regents of the University of California.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)mbuf.h 8.5 (Berkeley) 2/19/95
* $FreeBSD$
*/
#ifndef _SYS_MBUF_H_
#define _SYS_MBUF_H_
/* XXX: These includes suck. Sorry! */
#include <sys/queue.h>
#ifdef _KERNEL
#include <sys/systm.h>
#include <sys/refcount.h>
#include <vm/uma.h>
#ifdef WITNESS
#include <sys/lock.h>
#endif
#endif
#ifdef _KERNEL
#include <sys/sdt.h>
#define MBUF_PROBE1(probe, arg0) \
SDT_PROBE1(sdt, , , probe, arg0)
#define MBUF_PROBE2(probe, arg0, arg1) \
SDT_PROBE2(sdt, , , probe, arg0, arg1)
#define MBUF_PROBE3(probe, arg0, arg1, arg2) \
SDT_PROBE3(sdt, , , probe, arg0, arg1, arg2)
#define MBUF_PROBE4(probe, arg0, arg1, arg2, arg3) \
SDT_PROBE4(sdt, , , probe, arg0, arg1, arg2, arg3)
#define MBUF_PROBE5(probe, arg0, arg1, arg2, arg3, arg4) \
SDT_PROBE5(sdt, , , probe, arg0, arg1, arg2, arg3, arg4)
SDT_PROBE_DECLARE(sdt, , , m__init);
SDT_PROBE_DECLARE(sdt, , , m__gethdr_raw);
SDT_PROBE_DECLARE(sdt, , , m__gethdr);
SDT_PROBE_DECLARE(sdt, , , m__get_raw);
SDT_PROBE_DECLARE(sdt, , , m__get);
SDT_PROBE_DECLARE(sdt, , , m__getcl);
SDT_PROBE_DECLARE(sdt, , , m__getjcl);
SDT_PROBE_DECLARE(sdt, , , m__clget);
SDT_PROBE_DECLARE(sdt, , , m__cljget);
SDT_PROBE_DECLARE(sdt, , , m__cljset);
SDT_PROBE_DECLARE(sdt, , , m__free);
SDT_PROBE_DECLARE(sdt, , , m__freem);
#endif /* _KERNEL */
/*
* Mbufs are of a single size, MSIZE (sys/param.h), which includes overhead.
* An mbuf may add a single "mbuf cluster" of size MCLBYTES (also in
* sys/param.h), which has no additional overhead and is used instead of the
* internal data area; this is done when at least MINCLSIZE of data must be
* stored. Additionally, it is possible to allocate a separate buffer
* externally and attach it to the mbuf in a way similar to that of mbuf
* clusters.
*
* NB: These calculation do not take actual compiler-induced alignment and
* padding inside the complete struct mbuf into account. Appropriate
* attention is required when changing members of struct mbuf.
*
* MLEN is data length in a normal mbuf.
* MHLEN is data length in an mbuf with pktheader.
* MINCLSIZE is a smallest amount of data that should be put into cluster.
*
* Compile-time assertions in uipc_mbuf.c test these values to ensure that
* they are sensible.
*/
struct mbuf;
#define MHSIZE offsetof(struct mbuf, m_dat)
#define MPKTHSIZE offsetof(struct mbuf, m_pktdat)
#define MLEN ((int)(MSIZE - MHSIZE))
#define MHLEN ((int)(MSIZE - MPKTHSIZE))
#define MINCLSIZE (MHLEN + 1)
#define M_NODOM 255
#ifdef _KERNEL
/*-
* Macro for type conversion: convert mbuf pointer to data pointer of correct
* type:
*
* mtod(m, t) -- Convert mbuf pointer to data pointer of correct type.
* mtodo(m, o) -- Same as above but with offset 'o' into data.
*/
#define mtod(m, t) ((t)((m)->m_data))
#define mtodo(m, o) ((void *)(((m)->m_data) + (o)))
/*
* Argument structure passed to UMA routines during mbuf and packet
* allocations.
*/
struct mb_args {
int flags; /* Flags for mbuf being allocated */
short type; /* Type of mbuf being allocated */
};
#endif /* _KERNEL */
/*
* Packet tag structure (see below for details).
*/
struct m_tag {
SLIST_ENTRY(m_tag) m_tag_link; /* List of packet tags */
u_int16_t m_tag_id; /* Tag ID */
u_int16_t m_tag_len; /* Length of data */
u_int32_t m_tag_cookie; /* ABI/Module ID */
void (*m_tag_free)(struct m_tag *);
};
/*
* Static network interface owned tag.
* Allocated through ifp->if_snd_tag_alloc().
*/
struct if_snd_tag_sw;
struct m_snd_tag {
struct ifnet *ifp; /* network interface tag belongs to */
const struct if_snd_tag_sw *sw;
volatile u_int refcount;
};
/*
* Record/packet header in first mbuf of chain; valid only if M_PKTHDR is set.
* Size ILP32: 48
* LP64: 56
* Compile-time assertions in uipc_mbuf.c test these values to ensure that
* they are correct.
*/
struct pkthdr {
union {
struct m_snd_tag *snd_tag; /* send tag, if any */
struct ifnet *rcvif; /* rcv interface */
- struct {
- uint16_t rcvidx; /* rcv interface index ... */
- uint16_t rcvgen; /* ... and generation count */
- };
};
SLIST_HEAD(packet_tags, m_tag) tags; /* list of packet tags */
int32_t len; /* total packet length */
/* Layer crossing persistent information. */
uint32_t flowid; /* packet's 4-tuple system */
uint32_t csum_flags; /* checksum and offload features */
uint16_t fibnum; /* this packet should use this fib */
uint8_t numa_domain; /* NUMA domain of recvd pkt */
uint8_t rsstype; /* hash type */
union {
uint64_t rcv_tstmp; /* timestamp in ns */
struct {
uint8_t l2hlen; /* layer 2 hdr len */
uint8_t l3hlen; /* layer 3 hdr len */
uint8_t l4hlen; /* layer 4 hdr len */
uint8_t l5hlen; /* layer 5 hdr len */
uint8_t inner_l2hlen;
uint8_t inner_l3hlen;
uint8_t inner_l4hlen;
uint8_t inner_l5hlen;
};
};
union {
uint8_t eight[8];
uint16_t sixteen[4];
uint32_t thirtytwo[2];
uint64_t sixtyfour[1];
uintptr_t unintptr[1];
void *ptr;
} PH_per;
/* Layer specific non-persistent local storage for reassembly, etc. */
union {
uint8_t eight[8];
uint16_t sixteen[4];
uint32_t thirtytwo[2];
uint64_t sixtyfour[1];
uintptr_t unintptr[1];
void *ptr;
} PH_loc;
};
#define ether_vtag PH_per.sixteen[0]
#define tcp_tun_port PH_per.sixteen[0] /* outbound */
#define PH_vt PH_per
#define vt_nrecs sixteen[0] /* mld and v6-ND */
#define tso_segsz PH_per.sixteen[1] /* inbound after LRO */
#define lro_nsegs tso_segsz /* inbound after LRO */
#define csum_data PH_per.thirtytwo[1] /* inbound from hardware up */
#define lro_tcp_d_len PH_loc.sixteen[0] /* inbound during LRO (no reassembly) */
#define lro_tcp_d_csum PH_loc.sixteen[1] /* inbound during LRO (no reassembly) */
#define lro_tcp_h_off PH_loc.sixteen[2] /* inbound during LRO (no reassembly) */
#define lro_etype PH_loc.sixteen[3] /* inbound during LRO (no reassembly) */
/* Note PH_loc is used during IP reassembly (all 8 bytes as a ptr) */
/*
* TLS records for TLS 1.0-1.2 can have the following header lengths:
* - 5 (AES-CBC with implicit IV)
* - 21 (AES-CBC with explicit IV)
* - 13 (AES-GCM with 8 byte explicit IV)
*/
#define MBUF_PEXT_HDR_LEN 23
/*
* TLS records for TLS 1.0-1.2 can have the following maximum trailer
* lengths:
* - 16 (AES-GCM)
* - 36 (AES-CBC with SHA1 and up to 16 bytes of padding)
* - 48 (AES-CBC with SHA2-256 and up to 16 bytes of padding)
* - 64 (AES-CBC with SHA2-384 and up to 16 bytes of padding)
*/
#define MBUF_PEXT_TRAIL_LEN 64
#if defined(__LP64__)
#define MBUF_PEXT_MAX_PGS (40 / sizeof(vm_paddr_t))
#else
#define MBUF_PEXT_MAX_PGS (72 / sizeof(vm_paddr_t))
#endif
#define MBUF_PEXT_MAX_BYTES \
(MBUF_PEXT_MAX_PGS * PAGE_SIZE + MBUF_PEXT_HDR_LEN + MBUF_PEXT_TRAIL_LEN)
struct ktls_session;
struct socket;
/*
* Description of external storage mapped into mbuf; valid only if M_EXT is
* set.
* Size ILP32: 28
* LP64: 48
* Compile-time assertions in uipc_mbuf.c test these values to ensure that
* they are correct.
*/
typedef void m_ext_free_t(struct mbuf *);
struct m_ext {
union {
/*
* If EXT_FLAG_EMBREF is set, then we use refcount in the
* mbuf, the 'ext_count' member. Otherwise, we have a
* shadow copy and we use pointer 'ext_cnt'. The original
* mbuf is responsible to carry the pointer to free routine
* and its arguments. They aren't copied into shadows in
* mb_dupcl() to avoid dereferencing next cachelines.
*/
volatile u_int ext_count;
volatile u_int *ext_cnt;
};
uint32_t ext_size; /* size of buffer, for ext_free */
uint32_t ext_type:8, /* type of external storage */
ext_flags:24; /* external storage mbuf flags */
union {
struct {
/*
* Regular M_EXT mbuf:
* o ext_buf always points to the external buffer.
* o ext_free (below) and two optional arguments
* ext_arg1 and ext_arg2 store the free context for
* the external storage. They are set only in the
* refcount carrying mbuf, the one with
* EXT_FLAG_EMBREF flag, with exclusion for
* EXT_EXTREF type, where the free context is copied
* into all mbufs that use same external storage.
*/
char *ext_buf; /* start of buffer */
#define m_ext_copylen offsetof(struct m_ext, ext_arg2)
void *ext_arg2;
};
struct {
/*
* Multi-page M_EXTPG mbuf:
* o extpg_pa - page vector.
* o extpg_trail and extpg_hdr - TLS trailer and
* header.
* Uses ext_free and may also use ext_arg1.
*/
vm_paddr_t extpg_pa[MBUF_PEXT_MAX_PGS];
char extpg_trail[MBUF_PEXT_TRAIL_LEN];
char extpg_hdr[MBUF_PEXT_HDR_LEN];
/* Pretend these 3 fields are part of mbuf itself. */
#define m_epg_pa m_ext.extpg_pa
#define m_epg_trail m_ext.extpg_trail
#define m_epg_hdr m_ext.extpg_hdr
#define m_epg_ext_copylen offsetof(struct m_ext, ext_free)
};
};
/*
* Free method and optional argument pointer, both
* used by M_EXT and M_EXTPG.
*/
m_ext_free_t *ext_free;
void *ext_arg1;
};
/*
* The core of the mbuf object along with some shortcut defines for practical
* purposes.
*/
struct mbuf {
/*
* Header present at the beginning of every mbuf.
* Size ILP32: 24
* LP64: 32
* Compile-time assertions in uipc_mbuf.c test these values to ensure
* that they are correct.
*/
union { /* next buffer in chain */
struct mbuf *m_next;
SLIST_ENTRY(mbuf) m_slist;
STAILQ_ENTRY(mbuf) m_stailq;
};
union { /* next chain in queue/record */
struct mbuf *m_nextpkt;
SLIST_ENTRY(mbuf) m_slistpkt;
STAILQ_ENTRY(mbuf) m_stailqpkt;
};
caddr_t m_data; /* location of data */
int32_t m_len; /* amount of data in this mbuf */
uint32_t m_type:8, /* type of data in this mbuf */
m_flags:24; /* flags; see below */
#if !defined(__LP64__)
uint32_t m_pad; /* pad for 64bit alignment */
#endif
/*
* A set of optional headers (packet header, external storage header)
* and internal data storage. Historically, these arrays were sized
* to MHLEN (space left after a packet header) and MLEN (space left
* after only a regular mbuf header); they are now variable size in
* order to support future work on variable-size mbufs.
*/
union {
struct {
union {
/* M_PKTHDR set. */
struct pkthdr m_pkthdr;
/* M_EXTPG set.
* Multi-page M_EXTPG mbuf has its meta data
* split between the below anonymous structure
* and m_ext. It carries vector of pages,
* optional header and trailer char vectors
* and pointers to socket/TLS data.
*/
#define m_epg_startcopy m_epg_npgs
#define m_epg_endcopy m_epg_stailq
struct {
/* Overall count of pages and count of
* pages with I/O pending. */
uint8_t m_epg_npgs;
uint8_t m_epg_nrdy;
/* TLS header and trailer lengths.
* The data itself resides in m_ext. */
uint8_t m_epg_hdrlen;
uint8_t m_epg_trllen;
/* Offset into 1st page and length of
* data in the last page. */
uint16_t m_epg_1st_off;
uint16_t m_epg_last_len;
uint8_t m_epg_flags;
#define EPG_FLAG_ANON 0x1 /* Data can be encrypted in place. */
#define EPG_FLAG_2FREE 0x2 /* Scheduled for free. */
uint8_t m_epg_record_type;
uint8_t __spare[2];
int m_epg_enc_cnt;
struct ktls_session *m_epg_tls;
struct socket *m_epg_so;
uint64_t m_epg_seqno;
STAILQ_ENTRY(mbuf) m_epg_stailq;
};
};
union {
/* M_EXT or M_EXTPG set. */
struct m_ext m_ext;
/* M_PKTHDR set, neither M_EXT nor M_EXTPG. */
char m_pktdat[0];
};
};
char m_dat[0]; /* !M_PKTHDR, !M_EXT */
};
};
#ifdef _KERNEL
static inline int
m_epg_pagelen(const struct mbuf *m, int pidx, int pgoff)
{
KASSERT(pgoff == 0 || pidx == 0,
("page %d with non-zero offset %d in %p", pidx, pgoff, m));
if (pidx == m->m_epg_npgs - 1) {
return (m->m_epg_last_len);
} else {
return (PAGE_SIZE - pgoff);
}
}
#ifdef INVARIANTS
#define MCHECK(ex, msg) KASSERT((ex), \
("Multi page mbuf %p with " #msg " at %s:%d", \
m, __FILE__, __LINE__))
/*
* NB: This expects a non-empty buffer (npgs > 0 and
* last_pg_len > 0).
*/
#define MBUF_EXT_PGS_ASSERT_SANITY(m) do { \
MCHECK(m->m_epg_npgs > 0, "no valid pages"); \
MCHECK(m->m_epg_npgs <= nitems(m->m_epg_pa), \
"too many pages"); \
MCHECK(m->m_epg_nrdy <= m->m_epg_npgs, \
"too many ready pages"); \
MCHECK(m->m_epg_1st_off < PAGE_SIZE, \
"too large page offset"); \
MCHECK(m->m_epg_last_len > 0, "zero last page length"); \
MCHECK(m->m_epg_last_len <= PAGE_SIZE, \
"too large last page length"); \
if (m->m_epg_npgs == 1) \
MCHECK(m->m_epg_1st_off + \
m->m_epg_last_len <= PAGE_SIZE, \
"single page too large"); \
MCHECK(m->m_epg_hdrlen <= sizeof(m->m_epg_hdr), \
"too large header length"); \
MCHECK(m->m_epg_trllen <= sizeof(m->m_epg_trail), \
"too large header length"); \
} while (0)
#else
#define MBUF_EXT_PGS_ASSERT_SANITY(m) do {} while (0)
#endif
#endif
/*
* mbuf flags of global significance and layer crossing.
* Those of only protocol/layer specific significance are to be mapped
* to M_PROTO[1-11] and cleared at layer handoff boundaries.
* NB: Limited to the lower 24 bits.
*/
#define M_EXT 0x00000001 /* has associated external storage */
#define M_PKTHDR 0x00000002 /* start of record */
#define M_EOR 0x00000004 /* end of record */
#define M_RDONLY 0x00000008 /* associated data is marked read-only */
#define M_BCAST 0x00000010 /* send/received as link-level broadcast */
#define M_MCAST 0x00000020 /* send/received as link-level multicast */
#define M_PROMISC 0x00000040 /* packet was not for us */
#define M_VLANTAG 0x00000080 /* ether_vtag is valid */
#define M_EXTPG 0x00000100 /* has array of unmapped pages and TLS */
#define M_NOFREE 0x00000200 /* do not free mbuf, embedded in cluster */
#define M_TSTMP 0x00000400 /* rcv_tstmp field is valid */
#define M_TSTMP_HPREC 0x00000800 /* rcv_tstmp is high-prec, typically
hw-stamped on port (useful for IEEE 1588
and 802.1AS) */
#define M_TSTMP_LRO 0x00001000 /* Time LRO pushed in pkt is valid in (PH_loc) */
#define M_PROTO1 0x00002000 /* protocol-specific */
#define M_PROTO2 0x00004000 /* protocol-specific */
#define M_PROTO3 0x00008000 /* protocol-specific */
#define M_PROTO4 0x00010000 /* protocol-specific */
#define M_PROTO5 0x00020000 /* protocol-specific */
#define M_PROTO6 0x00040000 /* protocol-specific */
#define M_PROTO7 0x00080000 /* protocol-specific */
#define M_PROTO8 0x00100000 /* protocol-specific */
#define M_PROTO9 0x00200000 /* protocol-specific */
#define M_PROTO10 0x00400000 /* protocol-specific */
#define M_PROTO11 0x00800000 /* protocol-specific */
/*
* Flags to purge when crossing layers.
*/
#define M_PROTOFLAGS \
(M_PROTO1|M_PROTO2|M_PROTO3|M_PROTO4|M_PROTO5|M_PROTO6|M_PROTO7|M_PROTO8|\
M_PROTO9|M_PROTO10|M_PROTO11)
/*
* Flags preserved when copying m_pkthdr.
*/
#define M_COPYFLAGS \
(M_PKTHDR|M_EOR|M_RDONLY|M_BCAST|M_MCAST|M_PROMISC|M_VLANTAG|M_TSTMP| \
M_TSTMP_HPREC|M_TSTMP_LRO|M_PROTOFLAGS)
/*
* Flags preserved during demote.
*/
#define M_DEMOTEFLAGS \
(M_EXT | M_RDONLY | M_NOFREE | M_EXTPG)
/*
* Mbuf flag description for use with printf(9) %b identifier.
*/
#define M_FLAG_BITS \
"\20\1M_EXT\2M_PKTHDR\3M_EOR\4M_RDONLY\5M_BCAST\6M_MCAST" \
"\7M_PROMISC\10M_VLANTAG\11M_EXTPG\12M_NOFREE\13M_TSTMP\14M_TSTMP_HPREC\15M_TSTMP_LRO"
#define M_FLAG_PROTOBITS \
"\16M_PROTO1\17M_PROTO2\20M_PROTO3\21M_PROTO4" \
"\22M_PROTO5\23M_PROTO6\24M_PROTO7\25M_PROTO8\26M_PROTO9" \
"\27M_PROTO10\28M_PROTO11"
#define M_FLAG_PRINTF (M_FLAG_BITS M_FLAG_PROTOBITS)
/*
* Network interface cards are able to hash protocol fields (such as IPv4
* addresses and TCP port numbers) classify packets into flows. These flows
* can then be used to maintain ordering while delivering packets to the OS
* via parallel input queues, as well as to provide a stateless affinity
* model. NIC drivers can pass up the hash via m->m_pkthdr.flowid, and set
* m_flag fields to indicate how the hash should be interpreted by the
* network stack.
*
* Most NICs support RSS, which provides ordering and explicit affinity, and
* use the hash m_flag bits to indicate what header fields were covered by
* the hash. M_HASHTYPE_OPAQUE and M_HASHTYPE_OPAQUE_HASH can be set by non-
* RSS cards or configurations that provide an opaque flow identifier, allowing
* for ordering and distribution without explicit affinity. Additionally,
* M_HASHTYPE_OPAQUE_HASH indicates that the flow identifier has hash
* properties.
*
* The meaning of the IPV6_EX suffix:
* "o Home address from the home address option in the IPv6 destination
* options header. If the extension header is not present, use the Source
* IPv6 Address.
* o IPv6 address that is contained in the Routing-Header-Type-2 from the
* associated extension header. If the extension header is not present,
* use the Destination IPv6 Address."
* Quoted from:
* https://docs.microsoft.com/en-us/windows-hardware/drivers/network/rss-hashing-types#ndishashipv6ex
*/
#define M_HASHTYPE_HASHPROP 0x80 /* has hash properties */
#define M_HASHTYPE_INNER 0x40 /* calculated from inner headers */
#define M_HASHTYPE_HASH(t) (M_HASHTYPE_HASHPROP | (t))
/* Microsoft RSS standard hash types */
#define M_HASHTYPE_NONE 0
#define M_HASHTYPE_RSS_IPV4 M_HASHTYPE_HASH(1) /* IPv4 2-tuple */
#define M_HASHTYPE_RSS_TCP_IPV4 M_HASHTYPE_HASH(2) /* TCPv4 4-tuple */
#define M_HASHTYPE_RSS_IPV6 M_HASHTYPE_HASH(3) /* IPv6 2-tuple */
#define M_HASHTYPE_RSS_TCP_IPV6 M_HASHTYPE_HASH(4) /* TCPv6 4-tuple */
#define M_HASHTYPE_RSS_IPV6_EX M_HASHTYPE_HASH(5) /* IPv6 2-tuple +
* ext hdrs */
#define M_HASHTYPE_RSS_TCP_IPV6_EX M_HASHTYPE_HASH(6) /* TCPv6 4-tuple +
* ext hdrs */
#define M_HASHTYPE_RSS_UDP_IPV4 M_HASHTYPE_HASH(7) /* IPv4 UDP 4-tuple*/
#define M_HASHTYPE_RSS_UDP_IPV6 M_HASHTYPE_HASH(9) /* IPv6 UDP 4-tuple*/
#define M_HASHTYPE_RSS_UDP_IPV6_EX M_HASHTYPE_HASH(10)/* IPv6 UDP 4-tuple +
* ext hdrs */
#define M_HASHTYPE_OPAQUE 0x3f /* ordering, not affinity */
#define M_HASHTYPE_OPAQUE_HASH M_HASHTYPE_HASH(M_HASHTYPE_OPAQUE)
/* ordering+hash, not affinity*/
#define M_HASHTYPE_CLEAR(m) ((m)->m_pkthdr.rsstype = 0)
#define M_HASHTYPE_GET(m) ((m)->m_pkthdr.rsstype & ~M_HASHTYPE_INNER)
#define M_HASHTYPE_SET(m, v) ((m)->m_pkthdr.rsstype = (v))
#define M_HASHTYPE_TEST(m, v) (M_HASHTYPE_GET(m) == (v))
#define M_HASHTYPE_ISHASH(m) \
(((m)->m_pkthdr.rsstype & M_HASHTYPE_HASHPROP) != 0)
#define M_HASHTYPE_SETINNER(m) do { \
(m)->m_pkthdr.rsstype |= M_HASHTYPE_INNER; \
} while (0)
/*
* External mbuf storage buffer types.
*/
#define EXT_CLUSTER 1 /* mbuf cluster */
#define EXT_SFBUF 2 /* sendfile(2)'s sf_buf */
#define EXT_JUMBOP 3 /* jumbo cluster page sized */
#define EXT_JUMBO9 4 /* jumbo cluster 9216 bytes */
#define EXT_JUMBO16 5 /* jumbo cluster 16184 bytes */
#define EXT_PACKET 6 /* mbuf+cluster from packet zone */
#define EXT_MBUF 7 /* external mbuf reference */
#define EXT_RXRING 8 /* data in NIC receive ring */
#define EXT_VENDOR1 224 /* for vendor-internal use */
#define EXT_VENDOR2 225 /* for vendor-internal use */
#define EXT_VENDOR3 226 /* for vendor-internal use */
#define EXT_VENDOR4 227 /* for vendor-internal use */
#define EXT_EXP1 244 /* for experimental use */
#define EXT_EXP2 245 /* for experimental use */
#define EXT_EXP3 246 /* for experimental use */
#define EXT_EXP4 247 /* for experimental use */
#define EXT_NET_DRV 252 /* custom ext_buf provided by net driver(s) */
#define EXT_MOD_TYPE 253 /* custom module's ext_buf type */
#define EXT_DISPOSABLE 254 /* can throw this buffer away w/page flipping */
#define EXT_EXTREF 255 /* has externally maintained ext_cnt ptr */
/*
* Flags for external mbuf buffer types.
* NB: limited to the lower 24 bits.
*/
#define EXT_FLAG_EMBREF 0x000001 /* embedded ext_count */
#define EXT_FLAG_EXTREF 0x000002 /* external ext_cnt, notyet */
#define EXT_FLAG_NOFREE 0x000010 /* don't free mbuf to pool, notyet */
#define EXT_FLAG_VENDOR1 0x010000 /* These flags are vendor */
#define EXT_FLAG_VENDOR2 0x020000 /* or submodule specific, */
#define EXT_FLAG_VENDOR3 0x040000 /* not used by mbuf code. */
#define EXT_FLAG_VENDOR4 0x080000 /* Set/read by submodule. */
#define EXT_FLAG_EXP1 0x100000 /* for experimental use */
#define EXT_FLAG_EXP2 0x200000 /* for experimental use */
#define EXT_FLAG_EXP3 0x400000 /* for experimental use */
#define EXT_FLAG_EXP4 0x800000 /* for experimental use */
/*
* EXT flag description for use with printf(9) %b identifier.
*/
#define EXT_FLAG_BITS \
"\20\1EXT_FLAG_EMBREF\2EXT_FLAG_EXTREF\5EXT_FLAG_NOFREE" \
"\21EXT_FLAG_VENDOR1\22EXT_FLAG_VENDOR2\23EXT_FLAG_VENDOR3" \
"\24EXT_FLAG_VENDOR4\25EXT_FLAG_EXP1\26EXT_FLAG_EXP2\27EXT_FLAG_EXP3" \
"\30EXT_FLAG_EXP4"
/*
* Flags indicating checksum, segmentation and other offload work to be
* done, or already done, by hardware or lower layers. It is split into
* separate inbound and outbound flags.
*
* Outbound flags that are set by upper protocol layers requesting lower
* layers, or ideally the hardware, to perform these offloading tasks.
* For outbound packets this field and its flags can be directly tested
* against ifnet if_hwassist. Note that the outbound and the inbound flags do
* not collide right now but they could be allowed to (as long as the flags are
* scrubbed appropriately when the direction of an mbuf changes). CSUM_BITS
* would also have to split into CSUM_BITS_TX and CSUM_BITS_RX.
*
* CSUM_INNER_<x> is the same as CSUM_<x> but it applies to the inner frame.
* The CSUM_ENCAP_<x> bits identify the outer encapsulation.
*/
#define CSUM_IP 0x00000001 /* IP header checksum offload */
#define CSUM_IP_UDP 0x00000002 /* UDP checksum offload */
#define CSUM_IP_TCP 0x00000004 /* TCP checksum offload */
#define CSUM_IP_SCTP 0x00000008 /* SCTP checksum offload */
#define CSUM_IP_TSO 0x00000010 /* TCP segmentation offload */
#define CSUM_IP_ISCSI 0x00000020 /* iSCSI checksum offload */
#define CSUM_INNER_IP6_UDP 0x00000040
#define CSUM_INNER_IP6_TCP 0x00000080
#define CSUM_INNER_IP6_TSO 0x00000100
#define CSUM_IP6_UDP 0x00000200 /* UDP checksum offload */
#define CSUM_IP6_TCP 0x00000400 /* TCP checksum offload */
#define CSUM_IP6_SCTP 0x00000800 /* SCTP checksum offload */
#define CSUM_IP6_TSO 0x00001000 /* TCP segmentation offload */
#define CSUM_IP6_ISCSI 0x00002000 /* iSCSI checksum offload */
#define CSUM_INNER_IP 0x00004000
#define CSUM_INNER_IP_UDP 0x00008000
#define CSUM_INNER_IP_TCP 0x00010000
#define CSUM_INNER_IP_TSO 0x00020000
#define CSUM_ENCAP_VXLAN 0x00040000 /* VXLAN outer encapsulation */
#define CSUM_ENCAP_RSVD1 0x00080000
/* Inbound checksum support where the checksum was verified by hardware. */
#define CSUM_INNER_L3_CALC 0x00100000
#define CSUM_INNER_L3_VALID 0x00200000
#define CSUM_INNER_L4_CALC 0x00400000
#define CSUM_INNER_L4_VALID 0x00800000
#define CSUM_L3_CALC 0x01000000 /* calculated layer 3 csum */
#define CSUM_L3_VALID 0x02000000 /* checksum is correct */
#define CSUM_L4_CALC 0x04000000 /* calculated layer 4 csum */
#define CSUM_L4_VALID 0x08000000 /* checksum is correct */
#define CSUM_L5_CALC 0x10000000 /* calculated layer 5 csum */
#define CSUM_L5_VALID 0x20000000 /* checksum is correct */
#define CSUM_COALESCED 0x40000000 /* contains merged segments */
#define CSUM_SND_TAG 0x80000000 /* Packet header has send tag */
#define CSUM_FLAGS_TX (CSUM_IP | CSUM_IP_UDP | CSUM_IP_TCP | CSUM_IP_SCTP | \
CSUM_IP_TSO | CSUM_IP_ISCSI | CSUM_INNER_IP6_UDP | CSUM_INNER_IP6_TCP | \
CSUM_INNER_IP6_TSO | CSUM_IP6_UDP | CSUM_IP6_TCP | CSUM_IP6_SCTP | \
CSUM_IP6_TSO | CSUM_IP6_ISCSI | CSUM_INNER_IP | CSUM_INNER_IP_UDP | \
CSUM_INNER_IP_TCP | CSUM_INNER_IP_TSO | CSUM_ENCAP_VXLAN | \
CSUM_ENCAP_RSVD1 | CSUM_SND_TAG)
#define CSUM_FLAGS_RX (CSUM_INNER_L3_CALC | CSUM_INNER_L3_VALID | \
CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID | CSUM_L3_CALC | CSUM_L3_VALID | \
CSUM_L4_CALC | CSUM_L4_VALID | CSUM_L5_CALC | CSUM_L5_VALID | \
CSUM_COALESCED)
/*
* CSUM flag description for use with printf(9) %b identifier.
*/
#define CSUM_BITS \
"\20\1CSUM_IP\2CSUM_IP_UDP\3CSUM_IP_TCP\4CSUM_IP_SCTP\5CSUM_IP_TSO" \
"\6CSUM_IP_ISCSI\7CSUM_INNER_IP6_UDP\10CSUM_INNER_IP6_TCP" \
"\11CSUM_INNER_IP6_TSO\12CSUM_IP6_UDP\13CSUM_IP6_TCP\14CSUM_IP6_SCTP" \
"\15CSUM_IP6_TSO\16CSUM_IP6_ISCSI\17CSUM_INNER_IP\20CSUM_INNER_IP_UDP" \
"\21CSUM_INNER_IP_TCP\22CSUM_INNER_IP_TSO\23CSUM_ENCAP_VXLAN" \
"\24CSUM_ENCAP_RSVD1\25CSUM_INNER_L3_CALC\26CSUM_INNER_L3_VALID" \
"\27CSUM_INNER_L4_CALC\30CSUM_INNER_L4_VALID\31CSUM_L3_CALC" \
"\32CSUM_L3_VALID\33CSUM_L4_CALC\34CSUM_L4_VALID\35CSUM_L5_CALC" \
"\36CSUM_L5_VALID\37CSUM_COALESCED\40CSUM_SND_TAG"
/* CSUM flags compatibility mappings. */
#define CSUM_IP_CHECKED CSUM_L3_CALC
#define CSUM_IP_VALID CSUM_L3_VALID
#define CSUM_DATA_VALID CSUM_L4_VALID
#define CSUM_PSEUDO_HDR CSUM_L4_CALC
#define CSUM_SCTP_VALID CSUM_L4_VALID
#define CSUM_DELAY_DATA (CSUM_TCP|CSUM_UDP)
#define CSUM_DELAY_IP CSUM_IP /* Only v4, no v6 IP hdr csum */
#define CSUM_DELAY_DATA_IPV6 (CSUM_TCP_IPV6|CSUM_UDP_IPV6)
#define CSUM_DATA_VALID_IPV6 CSUM_DATA_VALID
#define CSUM_TCP CSUM_IP_TCP
#define CSUM_UDP CSUM_IP_UDP
#define CSUM_SCTP CSUM_IP_SCTP
#define CSUM_TSO (CSUM_IP_TSO|CSUM_IP6_TSO)
#define CSUM_INNER_TSO (CSUM_INNER_IP_TSO|CSUM_INNER_IP6_TSO)
#define CSUM_UDP_IPV6 CSUM_IP6_UDP
#define CSUM_TCP_IPV6 CSUM_IP6_TCP
#define CSUM_SCTP_IPV6 CSUM_IP6_SCTP
#define CSUM_TLS_MASK (CSUM_L5_CALC|CSUM_L5_VALID)
#define CSUM_TLS_DECRYPTED CSUM_L5_CALC
/*
* mbuf types describing the content of the mbuf (including external storage).
*/
#define MT_NOTMBUF 0 /* USED INTERNALLY ONLY! Object is not mbuf */
#define MT_DATA 1 /* dynamic (data) allocation */
#define MT_HEADER MT_DATA /* packet header, use M_PKTHDR instead */
#define MT_VENDOR1 4 /* for vendor-internal use */
#define MT_VENDOR2 5 /* for vendor-internal use */
#define MT_VENDOR3 6 /* for vendor-internal use */
#define MT_VENDOR4 7 /* for vendor-internal use */
#define MT_SONAME 8 /* socket name */
#define MT_EXP1 9 /* for experimental use */
#define MT_EXP2 10 /* for experimental use */
#define MT_EXP3 11 /* for experimental use */
#define MT_EXP4 12 /* for experimental use */
#define MT_CONTROL 14 /* extra-data protocol message */
#define MT_EXTCONTROL 15 /* control message with externalized contents */
#define MT_OOBDATA 16 /* expedited data */
#define MT_NOINIT 255 /* Not a type but a flag to allocate
a non-initialized mbuf */
/*
* String names of mbuf-related UMA(9) and malloc(9) types. Exposed to
* !_KERNEL so that monitoring tools can look up the zones with
* libmemstat(3).
*/
#define MBUF_MEM_NAME "mbuf"
#define MBUF_CLUSTER_MEM_NAME "mbuf_cluster"
#define MBUF_PACKET_MEM_NAME "mbuf_packet"
#define MBUF_JUMBOP_MEM_NAME "mbuf_jumbo_page"
#define MBUF_JUMBO9_MEM_NAME "mbuf_jumbo_9k"
#define MBUF_JUMBO16_MEM_NAME "mbuf_jumbo_16k"
#define MBUF_TAG_MEM_NAME "mbuf_tag"
#define MBUF_EXTREFCNT_MEM_NAME "mbuf_ext_refcnt"
#define MBUF_EXTPGS_MEM_NAME "mbuf_extpgs"
#ifdef _KERNEL
union if_snd_tag_alloc_params;
#ifdef WITNESS
#define MBUF_CHECKSLEEP(how) do { \
if (how == M_WAITOK) \
WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, \
"Sleeping in \"%s\"", __func__); \
} while (0)
#else
#define MBUF_CHECKSLEEP(how) do {} while (0)
#endif
/*
* Network buffer allocation API
*
* The rest of it is defined in kern/kern_mbuf.c
*/
extern uma_zone_t zone_mbuf;
extern uma_zone_t zone_clust;
extern uma_zone_t zone_pack;
extern uma_zone_t zone_jumbop;
extern uma_zone_t zone_jumbo9;
extern uma_zone_t zone_jumbo16;
extern uma_zone_t zone_extpgs;
void mb_dupcl(struct mbuf *, struct mbuf *);
void mb_free_ext(struct mbuf *);
void mb_free_extpg(struct mbuf *);
void mb_free_mext_pgs(struct mbuf *);
struct mbuf *mb_alloc_ext_pgs(int, m_ext_free_t);
struct mbuf *mb_alloc_ext_plus_pages(int, int);
struct mbuf *mb_mapped_to_unmapped(struct mbuf *, int, int, int,
struct mbuf **);
int mb_unmapped_compress(struct mbuf *m);
struct mbuf *mb_unmapped_to_ext(struct mbuf *m);
void mb_free_notready(struct mbuf *m, int count);
void m_adj(struct mbuf *, int);
void m_adj_decap(struct mbuf *, int);
int m_apply(struct mbuf *, int, int,
int (*)(void *, void *, u_int), void *);
int m_append(struct mbuf *, int, c_caddr_t);
void m_cat(struct mbuf *, struct mbuf *);
void m_catpkt(struct mbuf *, struct mbuf *);
int m_clget(struct mbuf *m, int how);
void *m_cljget(struct mbuf *m, int how, int size);
struct mbuf *m_collapse(struct mbuf *, int, int);
void m_copyback(struct mbuf *, int, int, c_caddr_t);
void m_copydata(const struct mbuf *, int, int, caddr_t);
struct mbuf *m_copym(struct mbuf *, int, int, int);
struct mbuf *m_copypacket(struct mbuf *, int);
void m_copy_pkthdr(struct mbuf *, struct mbuf *);
struct mbuf *m_copyup(struct mbuf *, int, int);
struct mbuf *m_defrag(struct mbuf *, int);
void m_demote_pkthdr(struct mbuf *);
void m_demote(struct mbuf *, int, int);
struct mbuf *m_devget(char *, int, int, struct ifnet *,
void (*)(char *, caddr_t, u_int));
void m_dispose_extcontrolm(struct mbuf *m);
struct mbuf *m_dup(const struct mbuf *, int);
int m_dup_pkthdr(struct mbuf *, const struct mbuf *, int);
void m_extadd(struct mbuf *, char *, u_int, m_ext_free_t,
void *, void *, int, int);
u_int m_fixhdr(struct mbuf *);
struct mbuf *m_fragment(struct mbuf *, int, int);
void m_freem(struct mbuf *);
void m_free_raw(struct mbuf *);
struct mbuf *m_get2(int, int, short, int);
struct mbuf *m_get3(int, int, short, int);
struct mbuf *m_getjcl(int, short, int, int);
struct mbuf *m_getm2(struct mbuf *, int, int, short, int);
struct mbuf *m_getptr(struct mbuf *, int, int *);
u_int m_length(struct mbuf *, struct mbuf **);
int m_mbuftouio(struct uio *, const struct mbuf *, int);
void m_move_pkthdr(struct mbuf *, struct mbuf *);
int m_pkthdr_init(struct mbuf *, int);
struct mbuf *m_prepend(struct mbuf *, int, int);
void m_print(const struct mbuf *, int);
struct mbuf *m_pulldown(struct mbuf *, int, int, int *);
struct mbuf *m_pullup(struct mbuf *, int);
int m_sanity(struct mbuf *, int);
struct mbuf *m_split(struct mbuf *, int, int);
struct mbuf *m_uiotombuf(struct uio *, int, int, int, int);
int m_unmapped_uiomove(const struct mbuf *, int, struct uio *,
int);
struct mbuf *m_unshare(struct mbuf *, int);
int m_snd_tag_alloc(struct ifnet *,
union if_snd_tag_alloc_params *, struct m_snd_tag **);
void m_snd_tag_init(struct m_snd_tag *, struct ifnet *,
const struct if_snd_tag_sw *);
void m_snd_tag_destroy(struct m_snd_tag *);
-void m_rcvif_serialize(struct mbuf *);
-struct ifnet *m_rcvif_restore(struct mbuf *);
static __inline int
m_gettype(int size)
{
int type;
switch (size) {
case MSIZE:
type = EXT_MBUF;
break;
case MCLBYTES:
type = EXT_CLUSTER;
break;
#if MJUMPAGESIZE != MCLBYTES
case MJUMPAGESIZE:
type = EXT_JUMBOP;
break;
#endif
case MJUM9BYTES:
type = EXT_JUMBO9;
break;
case MJUM16BYTES:
type = EXT_JUMBO16;
break;
default:
panic("%s: invalid cluster size %d", __func__, size);
}
return (type);
}
/*
* Associated an external reference counted buffer with an mbuf.
*/
static __inline void
m_extaddref(struct mbuf *m, char *buf, u_int size, u_int *ref_cnt,
m_ext_free_t freef, void *arg1, void *arg2)
{
KASSERT(ref_cnt != NULL, ("%s: ref_cnt not provided", __func__));
atomic_add_int(ref_cnt, 1);
m->m_flags |= M_EXT;
m->m_ext.ext_buf = buf;
m->m_ext.ext_cnt = ref_cnt;
m->m_data = m->m_ext.ext_buf;
m->m_ext.ext_size = size;
m->m_ext.ext_free = freef;
m->m_ext.ext_arg1 = arg1;
m->m_ext.ext_arg2 = arg2;
m->m_ext.ext_type = EXT_EXTREF;
m->m_ext.ext_flags = 0;
}
static __inline uma_zone_t
m_getzone(int size)
{
uma_zone_t zone;
switch (size) {
case MCLBYTES:
zone = zone_clust;
break;
#if MJUMPAGESIZE != MCLBYTES
case MJUMPAGESIZE:
zone = zone_jumbop;
break;
#endif
case MJUM9BYTES:
zone = zone_jumbo9;
break;
case MJUM16BYTES:
zone = zone_jumbo16;
break;
default:
panic("%s: invalid cluster size %d", __func__, size);
}
return (zone);
}
/*
* Initialize an mbuf with linear storage.
*
* Inline because the consumer text overhead will be roughly the same to
* initialize or call a function with this many parameters and M_PKTHDR
* should go away with constant propagation for !MGETHDR.
*/
static __inline int
m_init(struct mbuf *m, int how, short type, int flags)
{
int error;
m->m_next = NULL;
m->m_nextpkt = NULL;
m->m_data = m->m_dat;
m->m_len = 0;
m->m_flags = flags;
m->m_type = type;
if (flags & M_PKTHDR)
error = m_pkthdr_init(m, how);
else
error = 0;
MBUF_PROBE5(m__init, m, how, type, flags, error);
return (error);
}
static __inline struct mbuf *
m_get_raw(int how, short type)
{
struct mbuf *m;
struct mb_args args;
args.flags = 0;
args.type = type | MT_NOINIT;
m = uma_zalloc_arg(zone_mbuf, &args, how);
MBUF_PROBE3(m__get_raw, how, type, m);
return (m);
}
static __inline struct mbuf *
m_get(int how, short type)
{
struct mbuf *m;
struct mb_args args;
args.flags = 0;
args.type = type;
m = uma_zalloc_arg(zone_mbuf, &args, how);
MBUF_PROBE3(m__get, how, type, m);
return (m);
}
static __inline struct mbuf *
m_gethdr_raw(int how, short type)
{
struct mbuf *m;
struct mb_args args;
args.flags = M_PKTHDR;
args.type = type | MT_NOINIT;
m = uma_zalloc_arg(zone_mbuf, &args, how);
MBUF_PROBE3(m__gethdr_raw, how, type, m);
return (m);
}
static __inline struct mbuf *
m_gethdr(int how, short type)
{
struct mbuf *m;
struct mb_args args;
args.flags = M_PKTHDR;
args.type = type;
m = uma_zalloc_arg(zone_mbuf, &args, how);
MBUF_PROBE3(m__gethdr, how, type, m);
return (m);
}
static __inline struct mbuf *
m_getcl(int how, short type, int flags)
{
struct mbuf *m;
struct mb_args args;
args.flags = flags;
args.type = type;
m = uma_zalloc_arg(zone_pack, &args, how);
MBUF_PROBE4(m__getcl, how, type, flags, m);
return (m);
}
/*
* XXX: m_cljset() is a dangerous API. One must attach only a new,
* unreferenced cluster to an mbuf(9). It is not possible to assert
* that, so care can be taken only by users of the API.
*/
static __inline void
m_cljset(struct mbuf *m, void *cl, int type)
{
int size;
switch (type) {
case EXT_CLUSTER:
size = MCLBYTES;
break;
#if MJUMPAGESIZE != MCLBYTES
case EXT_JUMBOP:
size = MJUMPAGESIZE;
break;
#endif
case EXT_JUMBO9:
size = MJUM9BYTES;
break;
case EXT_JUMBO16:
size = MJUM16BYTES;
break;
default:
panic("%s: unknown cluster type %d", __func__, type);
break;
}
m->m_data = m->m_ext.ext_buf = cl;
m->m_ext.ext_free = m->m_ext.ext_arg1 = m->m_ext.ext_arg2 = NULL;
m->m_ext.ext_size = size;
m->m_ext.ext_type = type;
m->m_ext.ext_flags = EXT_FLAG_EMBREF;
m->m_ext.ext_count = 1;
m->m_flags |= M_EXT;
MBUF_PROBE3(m__cljset, m, cl, type);
}
static __inline void
m_chtype(struct mbuf *m, short new_type)
{
m->m_type = new_type;
}
static __inline void
m_clrprotoflags(struct mbuf *m)
{
while (m) {
m->m_flags &= ~M_PROTOFLAGS;
m = m->m_next;
}
}
static __inline struct mbuf *
m_last(struct mbuf *m)
{
while (m->m_next)
m = m->m_next;
return (m);
}
static inline u_int
m_extrefcnt(struct mbuf *m)
{
KASSERT(m->m_flags & M_EXT, ("%s: M_EXT missing", __func__));
return ((m->m_ext.ext_flags & EXT_FLAG_EMBREF) ? m->m_ext.ext_count :
*m->m_ext.ext_cnt);
}
/*
* mbuf, cluster, and external object allocation macros (for compatibility
* purposes).
*/
#define M_MOVE_PKTHDR(to, from) m_move_pkthdr((to), (from))
#define MGET(m, how, type) ((m) = m_get((how), (type)))
#define MGETHDR(m, how, type) ((m) = m_gethdr((how), (type)))
#define MCLGET(m, how) m_clget((m), (how))
#define MEXTADD(m, buf, size, free, arg1, arg2, flags, type) \
m_extadd((m), (char *)(buf), (size), (free), (arg1), (arg2), \
(flags), (type))
#define m_getm(m, len, how, type) \
m_getm2((m), (len), (how), (type), M_PKTHDR)
/*
* Evaluate TRUE if it's safe to write to the mbuf m's data region (this can
* be both the local data payload, or an external buffer area, depending on
* whether M_EXT is set).
*/
#define M_WRITABLE(m) (((m)->m_flags & (M_RDONLY | M_EXTPG)) == 0 && \
(!(((m)->m_flags & M_EXT)) || \
(m_extrefcnt(m) == 1)))
/* Check if the supplied mbuf has a packet header, or else panic. */
#define M_ASSERTPKTHDR(m) \
KASSERT((m) != NULL && (m)->m_flags & M_PKTHDR, \
("%s: no mbuf packet header!", __func__))
/* Check if the supplied mbuf has no send tag, or else panic. */
#define M_ASSERT_NO_SND_TAG(m) \
KASSERT((m) != NULL && (m)->m_flags & M_PKTHDR && \
((m)->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0, \
("%s: receive mbuf has send tag!", __func__))
/* Check if mbuf is multipage. */
#define M_ASSERTEXTPG(m) \
KASSERT(((m)->m_flags & (M_EXTPG|M_PKTHDR)) == M_EXTPG, \
("%s: m %p is not multipage!", __func__, m))
/*
* Ensure that the supplied mbuf is a valid, non-free mbuf.
*
* XXX: Broken at the moment. Need some UMA magic to make it work again.
*/
#define M_ASSERTVALID(m) \
KASSERT((((struct mbuf *)m)->m_flags & 0) == 0, \
("%s: attempted use of a free mbuf!", __func__))
/* Check whether any mbuf in the chain is unmapped. */
#ifdef INVARIANTS
#define M_ASSERTMAPPED(m) do { \
for (struct mbuf *__m = (m); __m != NULL; __m = __m->m_next) \
KASSERT((__m->m_flags & M_EXTPG) == 0, \
("%s: chain %p contains an unmapped mbuf", __func__, (m)));\
} while (0)
#else
#define M_ASSERTMAPPED(m) do {} while (0)
#endif
/*
* Return the address of the start of the buffer associated with an mbuf,
* handling external storage, packet-header mbufs, and regular data mbufs.
*/
#define M_START(m) \
(((m)->m_flags & M_EXTPG) ? NULL : \
((m)->m_flags & M_EXT) ? (m)->m_ext.ext_buf : \
((m)->m_flags & M_PKTHDR) ? &(m)->m_pktdat[0] : \
&(m)->m_dat[0])
/*
* Return the size of the buffer associated with an mbuf, handling external
* storage, packet-header mbufs, and regular data mbufs.
*/
#define M_SIZE(m) \
(((m)->m_flags & M_EXT) ? (m)->m_ext.ext_size : \
((m)->m_flags & M_PKTHDR) ? MHLEN : \
MLEN)
/*
* Set the m_data pointer of a newly allocated mbuf to place an object of the
* specified size at the end of the mbuf, longword aligned.
*
* NB: Historically, we had M_ALIGN(), MH_ALIGN(), and MEXT_ALIGN() as
* separate macros, each asserting that it was called at the proper moment.
* This required callers to themselves test the storage type and call the
* right one. Rather than require callers to be aware of those layout
* decisions, we centralize here.
*/
static __inline void
m_align(struct mbuf *m, int len)
{
#ifdef INVARIANTS
const char *msg = "%s: not a virgin mbuf";
#endif
int adjust;
KASSERT(m->m_data == M_START(m), (msg, __func__));
adjust = M_SIZE(m) - len;
m->m_data += adjust &~ (sizeof(long)-1);
}
#define M_ALIGN(m, len) m_align(m, len)
#define MH_ALIGN(m, len) m_align(m, len)
#define MEXT_ALIGN(m, len) m_align(m, len)
/*
* Compute the amount of space available before the current start of data in
* an mbuf.
*
* The M_WRITABLE() is a temporary, conservative safety measure: the burden
* of checking writability of the mbuf data area rests solely with the caller.
*
* NB: In previous versions, M_LEADINGSPACE() would only check M_WRITABLE()
* for mbufs with external storage. We now allow mbuf-embedded data to be
* read-only as well.
*/
#define M_LEADINGSPACE(m) \
(M_WRITABLE(m) ? ((m)->m_data - M_START(m)) : 0)
/*
* Compute the amount of space available after the end of data in an mbuf.
*
* The M_WRITABLE() is a temporary, conservative safety measure: the burden
* of checking writability of the mbuf data area rests solely with the caller.
*
* NB: In previous versions, M_TRAILINGSPACE() would only check M_WRITABLE()
* for mbufs with external storage. We now allow mbuf-embedded data to be
* read-only as well.
*/
#define M_TRAILINGSPACE(m) \
(M_WRITABLE(m) ? \
((M_START(m) + M_SIZE(m)) - ((m)->m_data + (m)->m_len)) : 0)
/*
* Arrange to prepend space of size plen to mbuf m. If a new mbuf must be
* allocated, how specifies whether to wait. If the allocation fails, the
* original mbuf chain is freed and m is set to NULL.
*/
#define M_PREPEND(m, plen, how) do { \
struct mbuf **_mmp = &(m); \
struct mbuf *_mm = *_mmp; \
int _mplen = (plen); \
int __mhow = (how); \
\
MBUF_CHECKSLEEP(how); \
if (M_LEADINGSPACE(_mm) >= _mplen) { \
_mm->m_data -= _mplen; \
_mm->m_len += _mplen; \
} else \
_mm = m_prepend(_mm, _mplen, __mhow); \
if (_mm != NULL && _mm->m_flags & M_PKTHDR) \
_mm->m_pkthdr.len += _mplen; \
*_mmp = _mm; \
} while (0)
/*
* Change mbuf to new type. This is a relatively expensive operation and
* should be avoided.
*/
#define MCHTYPE(m, t) m_chtype((m), (t))
/* Return the rcvif of a packet header. */
static __inline struct ifnet *
m_rcvif(struct mbuf *m)
{
M_ASSERTPKTHDR(m);
if (m->m_pkthdr.csum_flags & CSUM_SND_TAG)
return (NULL);
return (m->m_pkthdr.rcvif);
}
/* Length to m_copy to copy all. */
#define M_COPYALL 1000000000
extern int max_datalen; /* MHLEN - max_hdr */
extern int max_hdr; /* Largest link + protocol header */
extern int max_linkhdr; /* Largest link-level header */
extern int max_protohdr; /* Largest protocol header */
extern int nmbclusters; /* Maximum number of clusters */
extern bool mb_use_ext_pgs; /* Use ext_pgs for sendfile */
/*-
* Network packets may have annotations attached by affixing a list of
* "packet tags" to the pkthdr structure. Packet tags are dynamically
* allocated semi-opaque data structures that have a fixed header
* (struct m_tag) that specifies the size of the memory block and a
* <cookie,type> pair that identifies it. The cookie is a 32-bit unique
* unsigned value used to identify a module or ABI. By convention this value
* is chosen as the date+time that the module is created, expressed as the
* number of seconds since the epoch (e.g., using date -u +'%s'). The type
* value is an ABI/module-specific value that identifies a particular
* annotation and is private to the module. For compatibility with systems
* like OpenBSD that define packet tags w/o an ABI/module cookie, the value
* PACKET_ABI_COMPAT is used to implement m_tag_get and m_tag_find
* compatibility shim functions and several tag types are defined below.
* Users that do not require compatibility should use a private cookie value
* so that packet tag-related definitions can be maintained privately.
*
* Note that the packet tag returned by m_tag_alloc has the default memory
* alignment implemented by malloc. To reference private data one can use a
* construct like:
*
* struct m_tag *mtag = m_tag_alloc(...);
* struct foo *p = (struct foo *)(mtag+1);
*
* if the alignment of struct m_tag is sufficient for referencing members of
* struct foo. Otherwise it is necessary to embed struct m_tag within the
* private data structure to insure proper alignment; e.g.,
*
* struct foo {
* struct m_tag tag;
* ...
* };
* struct foo *p = (struct foo *) m_tag_alloc(...);
* struct m_tag *mtag = &p->tag;
*/
/*
* Persistent tags stay with an mbuf until the mbuf is reclaimed. Otherwise
* tags are expected to ``vanish'' when they pass through a network
* interface. For most interfaces this happens normally as the tags are
* reclaimed when the mbuf is free'd. However in some special cases
* reclaiming must be done manually. An example is packets that pass through
* the loopback interface. Also, one must be careful to do this when
* ``turning around'' packets (e.g., icmp_reflect).
*
* To mark a tag persistent bit-or this flag in when defining the tag id.
* The tag will then be treated as described above.
*/
#define MTAG_PERSISTENT 0x800
#define PACKET_TAG_NONE 0 /* Nadda */
/* Packet tags for use with PACKET_ABI_COMPAT. */
#define PACKET_TAG_IPSEC_IN_DONE 1 /* IPsec applied, in */
#define PACKET_TAG_IPSEC_OUT_DONE 2 /* IPsec applied, out */
#define PACKET_TAG_IPSEC_IN_CRYPTO_DONE 3 /* NIC IPsec crypto done */
#define PACKET_TAG_IPSEC_OUT_CRYPTO_NEEDED 4 /* NIC IPsec crypto req'ed */
#define PACKET_TAG_IPSEC_IN_COULD_DO_CRYPTO 5 /* NIC notifies IPsec */
#define PACKET_TAG_IPSEC_PENDING_TDB 6 /* Reminder to do IPsec */
#define PACKET_TAG_BRIDGE 7 /* Bridge processing done */
#define PACKET_TAG_GIF 8 /* GIF processing done */
#define PACKET_TAG_GRE 9 /* GRE processing done */
#define PACKET_TAG_IN_PACKET_CHECKSUM 10 /* NIC checksumming done */
#define PACKET_TAG_ENCAP 11 /* Encap. processing */
#define PACKET_TAG_IPSEC_SOCKET 12 /* IPSEC socket ref */
#define PACKET_TAG_IPSEC_HISTORY 13 /* IPSEC history */
#define PACKET_TAG_IPV6_INPUT 14 /* IPV6 input processing */
#define PACKET_TAG_DUMMYNET 15 /* dummynet info */
#define PACKET_TAG_DIVERT 17 /* divert info */
#define PACKET_TAG_IPFORWARD 18 /* ipforward info */
#define PACKET_TAG_MACLABEL (19 | MTAG_PERSISTENT) /* MAC label */
#define PACKET_TAG_PF 21 /* PF/ALTQ information */
#define PACKET_TAG_RTSOCKFAM 25 /* rtsock sa family */
#define PACKET_TAG_IPOPTIONS 27 /* Saved IP options */
#define PACKET_TAG_CARP 28 /* CARP info */
#define PACKET_TAG_IPSEC_NAT_T_PORTS 29 /* two uint16_t */
#define PACKET_TAG_ND_OUTGOING 30 /* ND outgoing */
/* Specific cookies and tags. */
/* Packet tag routines. */
struct m_tag *m_tag_alloc(uint32_t, uint16_t, int, int);
void m_tag_delete(struct mbuf *, struct m_tag *);
void m_tag_delete_chain(struct mbuf *, struct m_tag *);
void m_tag_free_default(struct m_tag *);
struct m_tag *m_tag_locate(struct mbuf *, uint32_t, uint16_t,
struct m_tag *);
struct m_tag *m_tag_copy(struct m_tag *, int);
int m_tag_copy_chain(struct mbuf *, const struct mbuf *, int);
void m_tag_delete_nonpersistent(struct mbuf *);
/*
* Initialize the list of tags associated with an mbuf.
*/
static __inline void
m_tag_init(struct mbuf *m)
{
SLIST_INIT(&m->m_pkthdr.tags);
}
/*
* Set up the contents of a tag. Note that this does not fill in the free
* method; the caller is expected to do that.
*
* XXX probably should be called m_tag_init, but that was already taken.
*/
static __inline void
m_tag_setup(struct m_tag *t, uint32_t cookie, uint16_t type, int len)
{
t->m_tag_id = type;
t->m_tag_len = len;
t->m_tag_cookie = cookie;
}
/*
* Reclaim resources associated with a tag.
*/
static __inline void
m_tag_free(struct m_tag *t)
{
(*t->m_tag_free)(t);
}
/*
* Return the first tag associated with an mbuf.
*/
static __inline struct m_tag *
m_tag_first(struct mbuf *m)
{
return (SLIST_FIRST(&m->m_pkthdr.tags));
}
/*
* Return the next tag in the list of tags associated with an mbuf.
*/
static __inline struct m_tag *
m_tag_next(struct mbuf *m __unused, struct m_tag *t)
{
return (SLIST_NEXT(t, m_tag_link));
}
/*
* Prepend a tag to the list of tags associated with an mbuf.
*/
static __inline void
m_tag_prepend(struct mbuf *m, struct m_tag *t)
{
SLIST_INSERT_HEAD(&m->m_pkthdr.tags, t, m_tag_link);
}
/*
* Unlink a tag from the list of tags associated with an mbuf.
*/
static __inline void
m_tag_unlink(struct mbuf *m, struct m_tag *t)
{
SLIST_REMOVE(&m->m_pkthdr.tags, t, m_tag, m_tag_link);
}
/* These are for OpenBSD compatibility. */
#define MTAG_ABI_COMPAT 0 /* compatibility ABI */
static __inline struct m_tag *
m_tag_get(uint16_t type, int length, int wait)
{
return (m_tag_alloc(MTAG_ABI_COMPAT, type, length, wait));
}
static __inline struct m_tag *
m_tag_find(struct mbuf *m, uint16_t type, struct m_tag *start)
{
return (SLIST_EMPTY(&m->m_pkthdr.tags) ? (struct m_tag *)NULL :
m_tag_locate(m, MTAG_ABI_COMPAT, type, start));
}
static inline struct m_snd_tag *
m_snd_tag_ref(struct m_snd_tag *mst)
{
refcount_acquire(&mst->refcount);
return (mst);
}
static inline void
m_snd_tag_rele(struct m_snd_tag *mst)
{
if (refcount_release(&mst->refcount))
m_snd_tag_destroy(mst);
}
static __inline struct mbuf *
m_free(struct mbuf *m)
{
struct mbuf *n = m->m_next;
MBUF_PROBE1(m__free, m);
if ((m->m_flags & (M_PKTHDR|M_NOFREE)) == (M_PKTHDR|M_NOFREE))
m_tag_delete_chain(m, NULL);
if (m->m_flags & M_PKTHDR && m->m_pkthdr.csum_flags & CSUM_SND_TAG)
m_snd_tag_rele(m->m_pkthdr.snd_tag);
if (m->m_flags & M_EXTPG)
mb_free_extpg(m);
else if (m->m_flags & M_EXT)
mb_free_ext(m);
else if ((m->m_flags & M_NOFREE) == 0)
uma_zfree(zone_mbuf, m);
return (n);
}
static __inline int
rt_m_getfib(struct mbuf *m)
{
KASSERT(m->m_flags & M_PKTHDR , ("Attempt to get FIB from non header mbuf."));
return (m->m_pkthdr.fibnum);
}
#define M_GETFIB(_m) rt_m_getfib(_m)
#define M_SETFIB(_m, _fib) do { \
KASSERT((_m)->m_flags & M_PKTHDR, ("Attempt to set FIB on non header mbuf.")); \
((_m)->m_pkthdr.fibnum) = (_fib); \
} while (0)
/* flags passed as first argument for "m_xxx_tcpip_hash()" */
#define MBUF_HASHFLAG_L2 (1 << 2)
#define MBUF_HASHFLAG_L3 (1 << 3)
#define MBUF_HASHFLAG_L4 (1 << 4)
/* mbuf hashing helper routines */
uint32_t m_ether_tcpip_hash_init(void);
uint32_t m_ether_tcpip_hash(const uint32_t, const struct mbuf *, uint32_t);
uint32_t m_infiniband_tcpip_hash_init(void);
uint32_t m_infiniband_tcpip_hash(const uint32_t, const struct mbuf *, uint32_t);
#ifdef MBUF_PROFILING
void m_profile(struct mbuf *m);
#define M_PROFILE(m) m_profile(m)
#else
#define M_PROFILE(m)
#endif
struct mbufq {
STAILQ_HEAD(, mbuf) mq_head;
int mq_len;
int mq_maxlen;
};
static inline void
mbufq_init(struct mbufq *mq, int maxlen)
{
STAILQ_INIT(&mq->mq_head);
mq->mq_maxlen = maxlen;
mq->mq_len = 0;
}
static inline struct mbuf *
mbufq_flush(struct mbufq *mq)
{
struct mbuf *m;
m = STAILQ_FIRST(&mq->mq_head);
STAILQ_INIT(&mq->mq_head);
mq->mq_len = 0;
return (m);
}
static inline void
mbufq_drain(struct mbufq *mq)
{
struct mbuf *m, *n;
n = mbufq_flush(mq);
while ((m = n) != NULL) {
n = STAILQ_NEXT(m, m_stailqpkt);
m_freem(m);
}
}
static inline struct mbuf *
mbufq_first(const struct mbufq *mq)
{
return (STAILQ_FIRST(&mq->mq_head));
}
static inline struct mbuf *
mbufq_last(const struct mbufq *mq)
{
return (STAILQ_LAST(&mq->mq_head, mbuf, m_stailqpkt));
}
static inline int
mbufq_full(const struct mbufq *mq)
{
return (mq->mq_maxlen > 0 && mq->mq_len >= mq->mq_maxlen);
}
static inline int
mbufq_len(const struct mbufq *mq)
{
return (mq->mq_len);
}
static inline int
mbufq_enqueue(struct mbufq *mq, struct mbuf *m)
{
if (mbufq_full(mq))
return (ENOBUFS);
STAILQ_INSERT_TAIL(&mq->mq_head, m, m_stailqpkt);
mq->mq_len++;
return (0);
}
static inline struct mbuf *
mbufq_dequeue(struct mbufq *mq)
{
struct mbuf *m;
m = STAILQ_FIRST(&mq->mq_head);
if (m) {
STAILQ_REMOVE_HEAD(&mq->mq_head, m_stailqpkt);
m->m_nextpkt = NULL;
mq->mq_len--;
}
return (m);
}
static inline void
mbufq_prepend(struct mbufq *mq, struct mbuf *m)
{
STAILQ_INSERT_HEAD(&mq->mq_head, m, m_stailqpkt);
mq->mq_len++;
}
/*
* Note: this doesn't enforce the maximum list size for dst.
*/
static inline void
mbufq_concat(struct mbufq *mq_dst, struct mbufq *mq_src)
{
mq_dst->mq_len += mq_src->mq_len;
STAILQ_CONCAT(&mq_dst->mq_head, &mq_src->mq_head);
mq_src->mq_len = 0;
}
#ifdef _SYS_TIMESPEC_H_
static inline void
mbuf_tstmp2timespec(struct mbuf *m, struct timespec *ts)
{
KASSERT((m->m_flags & M_PKTHDR) != 0, ("mbuf %p no M_PKTHDR", m));
KASSERT((m->m_flags & (M_TSTMP|M_TSTMP_LRO)) != 0, ("mbuf %p no M_TSTMP or M_TSTMP_LRO", m));
ts->tv_sec = m->m_pkthdr.rcv_tstmp / 1000000000;
ts->tv_nsec = m->m_pkthdr.rcv_tstmp % 1000000000;
}
#endif
#ifdef DEBUGNET
/* Invoked from the debugnet client code. */
void debugnet_mbuf_drain(void);
void debugnet_mbuf_start(void);
void debugnet_mbuf_finish(void);
void debugnet_mbuf_reinit(int nmbuf, int nclust, int clsize);
#endif
static inline bool
mbuf_has_tls_session(struct mbuf *m)
{
if (m->m_flags & M_EXTPG) {
if (m->m_epg_tls != NULL) {
return (true);
}
}
return (false);
}
#endif /* _KERNEL */
#endif /* !_SYS_MBUF_H_ */