Differential D26770 Diff 78446 head/sys/kern/subr_vmem.c

Changeset View

Standalone View

head/sys/kern/subr_vmem.c

Show First 20 Lines • Show All 362 Lines • ▼ Show 20 Lines	bt_free(vmem_t vm, bt_t bt)

VMEM_ASSERT_LOCKED(vm);		VMEM_ASSERT_LOCKED(vm);
MPASS(LIST_FIRST(&vm->vm_freetags) != bt);		MPASS(LIST_FIRST(&vm->vm_freetags) != bt);
LIST_INSERT_HEAD(&vm->vm_freetags, bt, bt_freelist);		LIST_INSERT_HEAD(&vm->vm_freetags, bt, bt_freelist);
vm->vm_nfreetags++;		vm->vm_nfreetags++;
}		}

/*		/*
		* Hide MAXALLOC tags before dropping the arena lock to ensure that a
		* concurrent allocation attempt does not grab them.
		*/
		static void
		bt_save(vmem_t *vm)
		{
		KASSERT(vm->vm_nfreetags >= BT_MAXALLOC,
		("%s: insufficient free tags %d", __func__, vm->vm_nfreetags));
		vm->vm_nfreetags -= BT_MAXALLOC;
		}

		static void
		bt_restore(vmem_t *vm)
		{
		vm->vm_nfreetags += BT_MAXALLOC;
		}

		/*
* freelist[0] ... [1, 1]		* freelist[0] ... [1, 1]
* freelist[1] ... [2, 2]		* freelist[1] ... [2, 2]
* :		* :
* freelist[29] ... [30, 30]		* freelist[29] ... [30, 30]
* freelist[30] ... [31, 31]		* freelist[30] ... [31, 31]
* freelist[31] ... [32, 63]		* freelist[31] ... [32, 63]
* freelist[33] ... [64, 127]		* freelist[33] ... [64, 127]
* :		* :
▲ Show 20 Lines • Show All 527 Lines • ▼ Show 20 Lines	vmem_import(vmem_t *vm, vmem_size_t size, vmem_size_t align, int flags)
*/		*/
if (align != vm->vm_quantum_mask + 1)		if (align != vm->vm_quantum_mask + 1)
size = (align * 2) + size;		size = (align * 2) + size;
size = roundup(size, vm->vm_import_quantum);		size = roundup(size, vm->vm_import_quantum);

if (vm->vm_limit != 0 && vm->vm_limit < vm->vm_size + size)		if (vm->vm_limit != 0 && vm->vm_limit < vm->vm_size + size)
return (ENOMEM);		return (ENOMEM);

/*		bt_save(vm);
* Hide MAXALLOC tags so we're guaranteed to be able to add this
* span and the tag we want to allocate from it.
*/
MPASS(vm->vm_nfreetags >= BT_MAXALLOC);
vm->vm_nfreetags -= BT_MAXALLOC;
VMEM_UNLOCK(vm);		VMEM_UNLOCK(vm);
error = (vm->vm_importfn)(vm->vm_arg, size, flags, &addr);		error = (vm->vm_importfn)(vm->vm_arg, size, flags, &addr);
VMEM_LOCK(vm);		VMEM_LOCK(vm);
vm->vm_nfreetags += BT_MAXALLOC;		bt_restore(vm);
if (error)		if (error)
return (ENOMEM);		return (ENOMEM);

vmem_add1(vm, addr, size, BT_TYPE_SPAN);		vmem_add1(vm, addr, size, BT_TYPE_SPAN);

return 0;		return 0;
}		}

▲ Show 20 Lines • Show All 111 Lines • ▼ Show 20 Lines	if (vmem_import(vm, size, align, flags) == 0)
return (1);		return (1);

/*		/*
* Try to free some space from the quantum cache or reclaim		* Try to free some space from the quantum cache or reclaim
* functions if available.		* functions if available.
*/		*/
if (vm->vm_qcache_max != 0 \|\| vm->vm_reclaimfn != NULL) {		if (vm->vm_qcache_max != 0 \|\| vm->vm_reclaimfn != NULL) {
avail = vm->vm_size - vm->vm_inuse;		avail = vm->vm_size - vm->vm_inuse;
		bt_save(vm);
VMEM_UNLOCK(vm);		VMEM_UNLOCK(vm);
if (vm->vm_qcache_max != 0)		if (vm->vm_qcache_max != 0)
qc_drain(vm);		qc_drain(vm);
if (vm->vm_reclaimfn != NULL)		if (vm->vm_reclaimfn != NULL)
vm->vm_reclaimfn(vm, flags);		vm->vm_reclaimfn(vm, flags);
VMEM_LOCK(vm);		VMEM_LOCK(vm);
		bt_restore(vm);
/* If we were successful retry even NOWAIT. */		/* If we were successful retry even NOWAIT. */
if (vm->vm_size - vm->vm_inuse > avail)		if (vm->vm_size - vm->vm_inuse > avail)
return (1);		return (1);
}		}
if ((flags & M_NOWAIT) != 0)		if ((flags & M_NOWAIT) != 0)
return (0);		return (0);
		bt_save(vm);
VMEM_CONDVAR_WAIT(vm);		VMEM_CONDVAR_WAIT(vm);
		bt_restore(vm);
return (1);		return (1);
}		}

static int		static int
vmem_try_release(vmem_t vm, struct vmem_btag bt, const bool remfree)		vmem_try_release(vmem_t vm, struct vmem_btag bt, const bool remfree)
{		{
struct vmem_btag *prev;		struct vmem_btag *prev;

Show All 31 Lines	vmem_xalloc_nextfit(vmem_t *vm, const vmem_size_t size, vmem_size_t align,
const vmem_size_t phase, const vmem_size_t nocross, int flags,		const vmem_size_t phase, const vmem_size_t nocross, int flags,
vmem_addr_t *addrp)		vmem_addr_t *addrp)
{		{
struct vmem_btag bt, cursor, next, prev;		struct vmem_btag bt, cursor, next, prev;
int error;		int error;

error = ENOMEM;		error = ENOMEM;
VMEM_LOCK(vm);		VMEM_LOCK(vm);
retry:
/*		/*
* Make sure we have enough tags to complete the operation.		* Make sure we have enough tags to complete the operation.
*/		*/
if (bt_fill(vm, flags) != 0)		if (bt_fill(vm, flags) != 0)
goto out;		goto out;

		retry:
/*		/*
* Find the next free tag meeting our constraints. If one is found,		* Find the next free tag meeting our constraints. If one is found,
* perform the allocation.		* perform the allocation.
*/		*/
for (cursor = &vm->vm_cursor, bt = TAILQ_NEXT(cursor, bt_seglist);		for (cursor = &vm->vm_cursor, bt = TAILQ_NEXT(cursor, bt_seglist);
bt != cursor; bt = TAILQ_NEXT(bt, bt_seglist)) {		bt != cursor; bt = TAILQ_NEXT(bt, bt_seglist)) {
if (bt == NULL)		if (bt == NULL)
bt = TAILQ_FIRST(&vm->vm_seglist);		bt = TAILQ_FIRST(&vm->vm_seglist);
▲ Show 20 Lines • Show All 259 Lines • ▼ Show 20 Lines	return (vmem_xalloc_nextfit(vm, size0, align, phase, nocross,
flags, addrp));		flags, addrp));

end = &vm->vm_freelist[VMEM_MAXORDER];		end = &vm->vm_freelist[VMEM_MAXORDER];
/*		/*
* choose a free block from which we allocate.		* choose a free block from which we allocate.
*/		*/
first = bt_freehead_toalloc(vm, size, strat);		first = bt_freehead_toalloc(vm, size, strat);
VMEM_LOCK(vm);		VMEM_LOCK(vm);
for (;;) {
/*		/*
* Make sure we have enough tags to complete the		* Make sure we have enough tags to complete the operation.
* operation.
*/		*/
error = bt_fill(vm, flags);		error = bt_fill(vm, flags);
if (error != 0)		if (error != 0)
break;		goto out;
		for (;;) {
/*		/*
* Scan freelists looking for a tag that satisfies the		* Scan freelists looking for a tag that satisfies the
* allocation. If we're doing BESTFIT we may encounter		* allocation. If we're doing BESTFIT we may encounter
* sizes below the request. If we're doing FIRSTFIT we		* sizes below the request. If we're doing FIRSTFIT we
* inspect only the first element from each list.		* inspect only the first element from each list.
*/		*/
for (list = first; list < end; list++) {		for (list = first; list < end; list++) {
LIST_FOREACH(bt, list, bt_freelist) {		LIST_FOREACH(bt, list, bt_freelist) {
▲ Show 20 Lines • Show All 404 Lines • Show Last 20 Lines