Differential D13187 Diff 35940 head/sys/kern/subr_vmem.c

Changeset View

Standalone View

head/sys/kern/subr_vmem.c

Show First 20 Lines • Show All 131 Lines • ▼ Show 20 Lines	struct vmem {
int vm_quantum_shift;		int vm_quantum_shift;

/* Written on alloc/free */		/* Written on alloc/free */
LIST_HEAD(, vmem_btag) vm_freetags;		LIST_HEAD(, vmem_btag) vm_freetags;
int vm_nfreetags;		int vm_nfreetags;
int vm_nbusytag;		int vm_nbusytag;
vmem_size_t vm_inuse;		vmem_size_t vm_inuse;
vmem_size_t vm_size;		vmem_size_t vm_size;
		vmem_size_t vm_limit;

/* Used on import. */		/* Used on import. */
vmem_import_t *vm_importfn;		vmem_import_t *vm_importfn;
vmem_release_t *vm_releasefn;		vmem_release_t *vm_releasefn;
void *vm_arg;		void *vm_arg;

/* Space exhaustion callback. */		/* Space exhaustion callback. */
vmem_reclaim_t *vm_reclaimfn;		vmem_reclaim_t *vm_reclaimfn;
▲ Show 20 Lines • Show All 75 Lines • ▼ Show 20 Lines
*/		*/
#define BT_MAXFREE (BT_MAXALLOC * 8)		#define BT_MAXFREE (BT_MAXALLOC * 8)

/* Allocator for boundary tags. */		/* Allocator for boundary tags. */
static uma_zone_t vmem_bt_zone;		static uma_zone_t vmem_bt_zone;

/* boot time arena storage. */		/* boot time arena storage. */
static struct vmem kernel_arena_storage;		static struct vmem kernel_arena_storage;
static struct vmem kmem_arena_storage;
static struct vmem buffer_arena_storage;		static struct vmem buffer_arena_storage;
static struct vmem transient_arena_storage;		static struct vmem transient_arena_storage;
		/* kernel and kmem arenas are aliased for backwards KPI compat. */
vmem_t *kernel_arena = &kernel_arena_storage;		vmem_t *kernel_arena = &kernel_arena_storage;
vmem_t *kmem_arena = &kmem_arena_storage;		vmem_t *kmem_arena = &kernel_arena_storage;
vmem_t *buffer_arena = &buffer_arena_storage;		vmem_t *buffer_arena = &buffer_arena_storage;
vmem_t *transient_arena = &transient_arena_storage;		vmem_t *transient_arena = &transient_arena_storage;

#ifdef DEBUG_MEMGUARD		#ifdef DEBUG_MEMGUARD
static struct vmem memguard_arena_storage;		static struct vmem memguard_arena_storage;
vmem_t *memguard_arena = &memguard_arena_storage;		vmem_t *memguard_arena = &memguard_arena_storage;
#endif		#endif

/*		/*
* Fill the vmem's boundary tag cache. We guarantee that boundary tag		* Fill the vmem's boundary tag cache. We guarantee that boundary tag
* allocation will not fail once bt_fill() passes. To do so we cache		* allocation will not fail once bt_fill() passes. To do so we cache
* at least the maximum possible tag allocations in the arena.		* at least the maximum possible tag allocations in the arena.
*/		*/
static int		static int
bt_fill(vmem_t *vm, int flags)		bt_fill(vmem_t *vm, int flags)
{		{
bt_t *bt;		bt_t *bt;

VMEM_ASSERT_LOCKED(vm);		VMEM_ASSERT_LOCKED(vm);

/*		/*
* Only allow the kmem arena to dip into reserve tags. It is the		* Only allow the kernel arena to dip into reserve tags. It is the
* vmem where new tags come from.		* vmem where new tags come from.
*/		*/
flags &= BT_FLAGS;		flags &= BT_FLAGS;
if (vm != kmem_arena)		if (vm != kernel_arena)
flags &= ~M_USE_RESERVE;		flags &= ~M_USE_RESERVE;

/*		/*
* Loop until we meet the reserve. To minimize the lock shuffle		* Loop until we meet the reserve. To minimize the lock shuffle
* and prevent simultaneous fills we first try a NOWAIT regardless		* and prevent simultaneous fills we first try a NOWAIT regardless
* of the caller's flags. Specify M_NOVM so we don't recurse while		* of the caller's flags. Specify M_NOVM so we don't recurse while
* holding a vmem lock.		* holding a vmem lock.
*/		*/
▲ Show 20 Lines • Show All 340 Lines • ▼ Show 20 Lines
* fills to proceed concurrently so NOWAIT is less likely to fail unless		* fills to proceed concurrently so NOWAIT is less likely to fail unless
* we are really out of KVA.		* we are really out of KVA.
*/		*/
static void *		static void *
vmem_bt_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *pflag, int wait)		vmem_bt_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *pflag, int wait)
{		{
vmem_addr_t addr;		vmem_addr_t addr;

*pflag = UMA_SLAB_KMEM;		*pflag = UMA_SLAB_KERNEL;

/*		/*
* Single thread boundary tag allocation so that the address space		* Single thread boundary tag allocation so that the address space
* and memory are added in one atomic operation.		* and memory are added in one atomic operation.
*/		*/
mtx_lock(&vmem_bt_lock);		mtx_lock(&vmem_bt_lock);
if (vmem_xalloc(kmem_arena, bytes, 0, 0, 0, VMEM_ADDR_MIN,		if (vmem_xalloc(kernel_arena, bytes, 0, 0, 0, VMEM_ADDR_MIN,
VMEM_ADDR_MAX, M_NOWAIT \| M_NOVM \| M_USE_RESERVE \| M_BESTFIT,		VMEM_ADDR_MAX, M_NOWAIT \| M_NOVM \| M_USE_RESERVE \| M_BESTFIT,
&addr) == 0) {		&addr) == 0) {
if (kmem_back(kmem_object, addr, bytes,		if (kmem_back(kernel_object, addr, bytes,
M_NOWAIT \| M_USE_RESERVE) == 0) {		M_NOWAIT \| M_USE_RESERVE) == 0) {
mtx_unlock(&vmem_bt_lock);		mtx_unlock(&vmem_bt_lock);
return ((void *)addr);		return ((void *)addr);
}		}
vmem_xfree(kmem_arena, addr, bytes);		vmem_xfree(kernel_arena, addr, bytes);
mtx_unlock(&vmem_bt_lock);		mtx_unlock(&vmem_bt_lock);
/*		/*
* Out of memory, not address space. This may not even be		* Out of memory, not address space. This may not even be
* possible due to M_USE_RESERVE page allocation.		* possible due to M_USE_RESERVE page allocation.
*/		*/
if (wait & M_WAITOK)		if (wait & M_WAITOK)
VM_WAIT;		VM_WAIT;
return (NULL);		return (NULL);
▲ Show 20 Lines • Show All 188 Lines • ▼ Show 20 Lines

static int		static int
vmem_import(vmem_t *vm, vmem_size_t size, vmem_size_t align, int flags)		vmem_import(vmem_t *vm, vmem_size_t size, vmem_size_t align, int flags)
{		{
vmem_addr_t addr;		vmem_addr_t addr;
int error;		int error;

if (vm->vm_importfn == NULL)		if (vm->vm_importfn == NULL)
return EINVAL;		return (EINVAL);

/*		/*
* To make sure we get a span that meets the alignment we double it		* To make sure we get a span that meets the alignment we double it
* and add the size to the tail. This slightly overestimates.		* and add the size to the tail. This slightly overestimates.
*/		*/
if (align != vm->vm_quantum_mask + 1)		if (align != vm->vm_quantum_mask + 1)
size = (align * 2) + size;		size = (align * 2) + size;
size = roundup(size, vm->vm_import_quantum);		size = roundup(size, vm->vm_import_quantum);

		if (vm->vm_limit != 0 && vm->vm_limit < vm->vm_size + size)
		return (ENOMEM);

/*		/*
* Hide MAXALLOC tags so we're guaranteed to be able to add this		* Hide MAXALLOC tags so we're guaranteed to be able to add this
* span and the tag we want to allocate from it.		* span and the tag we want to allocate from it.
*/		*/
MPASS(vm->vm_nfreetags >= BT_MAXALLOC);		MPASS(vm->vm_nfreetags >= BT_MAXALLOC);
vm->vm_nfreetags -= BT_MAXALLOC;		vm->vm_nfreetags -= BT_MAXALLOC;
VMEM_UNLOCK(vm);		VMEM_UNLOCK(vm);
error = (vm->vm_importfn)(vm->vm_arg, size, flags, &addr);		error = (vm->vm_importfn)(vm->vm_arg, size, flags, &addr);
VMEM_LOCK(vm);		VMEM_LOCK(vm);
vm->vm_nfreetags += BT_MAXALLOC;		vm->vm_nfreetags += BT_MAXALLOC;
if (error)		if (error)
return ENOMEM;		return (ENOMEM);

vmem_add1(vm, addr, size, BT_TYPE_SPAN);		vmem_add1(vm, addr, size, BT_TYPE_SPAN);

return 0;		return 0;
}		}

/*		/*
* vmem_fit: check if a bt can satisfy the given restrictions.		* vmem_fit: check if a bt can satisfy the given restrictions.
▲ Show 20 Lines • Show All 105 Lines • ▼ Show 20 Lines	vmem_set_import(vmem_t vm, vmem_import_t importfn,
vm->vm_importfn = importfn;		vm->vm_importfn = importfn;
vm->vm_releasefn = releasefn;		vm->vm_releasefn = releasefn;
vm->vm_arg = arg;		vm->vm_arg = arg;
vm->vm_import_quantum = import_quantum;		vm->vm_import_quantum = import_quantum;
VMEM_UNLOCK(vm);		VMEM_UNLOCK(vm);
}		}

void		void
		vmem_set_limit(vmem_t *vm, vmem_size_t limit)
		{

		VMEM_LOCK(vm);
		vm->vm_limit = limit;
		VMEM_UNLOCK(vm);
		}

		void
vmem_set_reclaim(vmem_t vm, vmem_reclaim_t reclaimfn)		vmem_set_reclaim(vmem_t vm, vmem_reclaim_t reclaimfn)
{		{

VMEM_LOCK(vm);		VMEM_LOCK(vm);
vm->vm_reclaimfn = reclaimfn;		vm->vm_reclaimfn = reclaimfn;
VMEM_UNLOCK(vm);		VMEM_UNLOCK(vm);
}		}

Show All 15 Lines	vmem_init(vmem_t vm, const char name, vmem_addr_t base, vmem_size_t size,
VMEM_LOCK_INIT(vm, name);		VMEM_LOCK_INIT(vm, name);
vm->vm_nfreetags = 0;		vm->vm_nfreetags = 0;
LIST_INIT(&vm->vm_freetags);		LIST_INIT(&vm->vm_freetags);
strlcpy(vm->vm_name, name, sizeof(vm->vm_name));		strlcpy(vm->vm_name, name, sizeof(vm->vm_name));
vm->vm_quantum_mask = quantum - 1;		vm->vm_quantum_mask = quantum - 1;
vm->vm_quantum_shift = flsl(quantum) - 1;		vm->vm_quantum_shift = flsl(quantum) - 1;
vm->vm_nbusytag = 0;		vm->vm_nbusytag = 0;
vm->vm_size = 0;		vm->vm_size = 0;
		vm->vm_limit = 0;
vm->vm_inuse = 0;		vm->vm_inuse = 0;
qc_init(vm, qcache_max);		qc_init(vm, qcache_max);

TAILQ_INIT(&vm->vm_seglist);		TAILQ_INIT(&vm->vm_seglist);
for (i = 0; i < VMEM_MAXORDER; i++) {		for (i = 0; i < VMEM_MAXORDER; i++) {
LIST_INIT(&vm->vm_freelist[i]);		LIST_INIT(&vm->vm_freelist[i]);
}		}
memset(&vm->vm_hash0, 0, sizeof(vm->vm_hash0));		memset(&vm->vm_hash0, 0, sizeof(vm->vm_hash0));
▲ Show 20 Lines • Show All 569 Lines • Show Last 20 Lines