Differential D3864 Diff 11470 head/sys/kern/kern_mbuf.c

Changeset View

Standalone View

head/sys/kern/kern_mbuf.c

Show All 26 Lines

#include <sys/cdefs.h>		#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");		__FBSDID("$FreeBSD$");

#include "opt_param.h"		#include "opt_param.h"

#include <sys/param.h>		#include <sys/param.h>
#include <sys/malloc.h>		#include <sys/malloc.h>
		#include <sys/types.h>
#include <sys/systm.h>		#include <sys/systm.h>
#include <sys/mbuf.h>		#include <sys/mbuf.h>
#include <sys/domain.h>		#include <sys/domain.h>
#include <sys/eventhandler.h>		#include <sys/eventhandler.h>
#include <sys/kernel.h>		#include <sys/kernel.h>
		#include <sys/lock.h>
		#include <sys/mutex.h>
#include <sys/protosw.h>		#include <sys/protosw.h>
#include <sys/smp.h>		#include <sys/smp.h>
#include <sys/sysctl.h>		#include <sys/sysctl.h>

#include <security/mac/mac_framework.h>		#include <security/mac/mac_framework.h>

#include <vm/vm.h>		#include <vm/vm.h>
#include <vm/vm_extern.h>		#include <vm/vm_extern.h>
▲ Show 20 Lines • Show All 219 Lines • ▼ Show 20 Lines
uma_zone_t zone_clust;		uma_zone_t zone_clust;
uma_zone_t zone_pack;		uma_zone_t zone_pack;
uma_zone_t zone_jumbop;		uma_zone_t zone_jumbop;
uma_zone_t zone_jumbo9;		uma_zone_t zone_jumbo9;
uma_zone_t zone_jumbo16;		uma_zone_t zone_jumbo16;
uma_zone_t zone_ext_refcnt;		uma_zone_t zone_ext_refcnt;

/*		/*
		* Callout to assist us in freeing mbufs.
		*/
		static struct callout mb_reclaim_callout;
		static struct mtx mb_reclaim_callout_mtx;

		/*
* Local prototypes.		* Local prototypes.
*/		*/
static int mb_ctor_mbuf(void , int, void , int);		static int mb_ctor_mbuf(void , int, void , int);
static int mb_ctor_clust(void , int, void , int);		static int mb_ctor_clust(void , int, void , int);
static int mb_ctor_pack(void , int, void , int);		static int mb_ctor_pack(void , int, void , int);
static void mb_dtor_mbuf(void , int, void );		static void mb_dtor_mbuf(void , int, void );
static void mb_dtor_clust(void , int, void );		static void mb_dtor_clust(void , int, void );
static void mb_dtor_pack(void , int, void );		static void mb_dtor_pack(void , int, void );
static int mb_zinit_pack(void *, int, int);		static int mb_zinit_pack(void *, int, int);
static void mb_zfini_pack(void *, int);		static void mb_zfini_pack(void *, int);

static void mb_reclaim(void *);		static void mb_reclaim(void *);
static void mbuf_jumbo_alloc(uma_zone_t, vm_size_t, uint8_t , int);		static void mbuf_jumbo_alloc(uma_zone_t, vm_size_t, uint8_t , int);
		static void mb_maxaction(uma_zone_t);

/* Ensure that MSIZE is a power of 2. */		/* Ensure that MSIZE is a power of 2. */
CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);		CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);

/*		/*
* Initialize FreeBSD Network buffer allocation.		* Initialize FreeBSD Network buffer allocation.
*/		*/
static void		static void
Show All 9 Lines	#ifdef INVARIANTS
trash_init, trash_fini,		trash_init, trash_fini,
#else		#else
NULL, NULL,		NULL, NULL,
#endif		#endif
MSIZE - 1, UMA_ZONE_MAXBUCKET);		MSIZE - 1, UMA_ZONE_MAXBUCKET);
if (nmbufs > 0)		if (nmbufs > 0)
nmbufs = uma_zone_set_max(zone_mbuf, nmbufs);		nmbufs = uma_zone_set_max(zone_mbuf, nmbufs);
uma_zone_set_warning(zone_mbuf, "kern.ipc.nmbufs limit reached");		uma_zone_set_warning(zone_mbuf, "kern.ipc.nmbufs limit reached");
		uma_zone_set_maxaction(zone_mbuf, mb_maxaction);

zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES,		zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES,
mb_ctor_clust, mb_dtor_clust,		mb_ctor_clust, mb_dtor_clust,
#ifdef INVARIANTS		#ifdef INVARIANTS
trash_init, trash_fini,		trash_init, trash_fini,
#else		#else
NULL, NULL,		NULL, NULL,
#endif		#endif
UMA_ALIGN_PTR, UMA_ZONE_REFCNT);		UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
if (nmbclusters > 0)		if (nmbclusters > 0)
nmbclusters = uma_zone_set_max(zone_clust, nmbclusters);		nmbclusters = uma_zone_set_max(zone_clust, nmbclusters);
uma_zone_set_warning(zone_clust, "kern.ipc.nmbclusters limit reached");		uma_zone_set_warning(zone_clust, "kern.ipc.nmbclusters limit reached");
		uma_zone_set_maxaction(zone_clust, mb_maxaction);

zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,		zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,
mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf);		mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf);

/* Make jumbo frame zone too. Page size, 9k and 16k. */		/* Make jumbo frame zone too. Page size, 9k and 16k. */
zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE,		zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE,
mb_ctor_clust, mb_dtor_clust,		mb_ctor_clust, mb_dtor_clust,
#ifdef INVARIANTS		#ifdef INVARIANTS
trash_init, trash_fini,		trash_init, trash_fini,
#else		#else
NULL, NULL,		NULL, NULL,
#endif		#endif
UMA_ALIGN_PTR, UMA_ZONE_REFCNT);		UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
if (nmbjumbop > 0)		if (nmbjumbop > 0)
nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop);		nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop);
uma_zone_set_warning(zone_jumbop, "kern.ipc.nmbjumbop limit reached");		uma_zone_set_warning(zone_jumbop, "kern.ipc.nmbjumbop limit reached");
		uma_zone_set_maxaction(zone_jumbop, mb_maxaction);

zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES,		zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES,
mb_ctor_clust, mb_dtor_clust,		mb_ctor_clust, mb_dtor_clust,
#ifdef INVARIANTS		#ifdef INVARIANTS
trash_init, trash_fini,		trash_init, trash_fini,
#else		#else
NULL, NULL,		NULL, NULL,
#endif		#endif
UMA_ALIGN_PTR, UMA_ZONE_REFCNT);		UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
uma_zone_set_allocf(zone_jumbo9, mbuf_jumbo_alloc);		uma_zone_set_allocf(zone_jumbo9, mbuf_jumbo_alloc);
if (nmbjumbo9 > 0)		if (nmbjumbo9 > 0)
nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9);		nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9);
uma_zone_set_warning(zone_jumbo9, "kern.ipc.nmbjumbo9 limit reached");		uma_zone_set_warning(zone_jumbo9, "kern.ipc.nmbjumbo9 limit reached");
		uma_zone_set_maxaction(zone_jumbo9, mb_maxaction);

zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES,		zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES,
mb_ctor_clust, mb_dtor_clust,		mb_ctor_clust, mb_dtor_clust,
#ifdef INVARIANTS		#ifdef INVARIANTS
trash_init, trash_fini,		trash_init, trash_fini,
#else		#else
NULL, NULL,		NULL, NULL,
#endif		#endif
UMA_ALIGN_PTR, UMA_ZONE_REFCNT);		UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
uma_zone_set_allocf(zone_jumbo16, mbuf_jumbo_alloc);		uma_zone_set_allocf(zone_jumbo16, mbuf_jumbo_alloc);
if (nmbjumbo16 > 0)		if (nmbjumbo16 > 0)
nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16);		nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16);
uma_zone_set_warning(zone_jumbo16, "kern.ipc.nmbjumbo16 limit reached");		uma_zone_set_warning(zone_jumbo16, "kern.ipc.nmbjumbo16 limit reached");
		uma_zone_set_maxaction(zone_jumbo16, mb_maxaction);

zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int),		zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int),
NULL, NULL,		NULL, NULL,
NULL, NULL,		NULL, NULL,
UMA_ALIGN_PTR, UMA_ZONE_ZINIT);		UMA_ALIGN_PTR, UMA_ZONE_ZINIT);

/* uma_prealloc() goes here... */		/* uma_prealloc() goes here... */

		/* Initialize the mb_reclaim() callout. */
		mtx_init(&mb_reclaim_callout_mtx, "mb_reclaim_callout_mtx", NULL,
		MTX_DEF);
		callout_init(&mb_reclaim_callout, 1);

/*		/*
* Hook event handler for low-memory situation, used to		* Hook event handler for low-memory situation, used to
* drain protocols and push data back to the caches (UMA		* drain protocols and push data back to the caches (UMA
* later pushes it back to VM).		* later pushes it back to VM).
*/		*/
EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL,		EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL,
EVENTHANDLER_PRI_FIRST);		EVENTHANDLER_PRI_FIRST);
}		}
▲ Show 20 Lines • Show All 289 Lines • ▼ Show 20 Lines	mb_reclaim(void *junk)

WITNESS_WARN(WARN_GIANTOK \| WARN_SLEEPOK \| WARN_PANIC, NULL,		WITNESS_WARN(WARN_GIANTOK \| WARN_SLEEPOK \| WARN_PANIC, NULL,
"mb_reclaim()");		"mb_reclaim()");

for (dp = domains; dp != NULL; dp = dp->dom_next)		for (dp = domains; dp != NULL; dp = dp->dom_next)
for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)		for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
if (pr->pr_drain != NULL)		if (pr->pr_drain != NULL)
(*pr->pr_drain)();		(*pr->pr_drain)();
		}

		/*
		* This is the function called by the mb_reclaim_callout, which is
		* used when we hit the maximum for a zone.
		*
		* (See mb_maxaction() below.)
		*/
		static void
		mb_reclaim_timer(void *junk __unused)
		{

		mtx_lock(&mb_reclaim_callout_mtx);

		/*
		* Avoid running this function extra times by skipping this invocation
		* if the callout has already been rescheduled.
		*/
		if (callout_pending(&mb_reclaim_callout) \|\|
		!callout_active(&mb_reclaim_callout)) {
		mtx_unlock(&mb_reclaim_callout_mtx);
		return;
		}
		mtx_unlock(&mb_reclaim_callout_mtx);

		mb_reclaim(NULL);

		mtx_lock(&mb_reclaim_callout_mtx);
		callout_deactivate(&mb_reclaim_callout);
		mtx_unlock(&mb_reclaim_callout_mtx);
		}

		/*
		* This function is called when we hit the maximum for a zone.
		*
		* At that point, we want to call the protocol drain routine to free up some
		* mbufs. However, we will use the callout routines to schedule this to
		* occur in another thread. (The thread calling this function holds the
		* zone lock.)
		*/
		static void
		mb_maxaction(uma_zone_t zone __unused)
		{

		/*
		* If we can't immediately obtain the lock, either the callout
		* is currently running, or another thread is scheduling the
		* callout.
		*/
		if (!mtx_trylock(&mb_reclaim_callout_mtx))
		return;

		/* If not already scheduled/running, schedule the callout. */
		if (!callout_active(&mb_reclaim_callout)) {
		callout_reset(&mb_reclaim_callout, 1, mb_reclaim_timer, NULL);
		}

		mtx_unlock(&mb_reclaim_callout_mtx);
}		}