Differential D22563 Diff 65041 head/sys/kern/kern_malloc.c

Changeset View

Standalone View

head/sys/kern/kern_malloc.c

Show First 20 Lines • Show All 54 Lines • ▼ Show 20 Lines
#include <sys/systm.h>		#include <sys/systm.h>
#include <sys/kdb.h>		#include <sys/kdb.h>
#include <sys/kernel.h>		#include <sys/kernel.h>
#include <sys/lock.h>		#include <sys/lock.h>
#include <sys/malloc.h>		#include <sys/malloc.h>
#include <sys/mutex.h>		#include <sys/mutex.h>
#include <sys/vmmeter.h>		#include <sys/vmmeter.h>
#include <sys/proc.h>		#include <sys/proc.h>
		#include <sys/queue.h>
#include <sys/sbuf.h>		#include <sys/sbuf.h>
#include <sys/smp.h>		#include <sys/smp.h>
#include <sys/sysctl.h>		#include <sys/sysctl.h>
#include <sys/time.h>		#include <sys/time.h>
#include <sys/vmem.h>		#include <sys/vmem.h>
#ifdef EPOCH_TRACE		#ifdef EPOCH_TRACE
#include <sys/epoch.h>		#include <sys/epoch.h>
#endif		#endif

#include <vm/vm.h>		#include <vm/vm.h>
#include <vm/pmap.h>		#include <vm/pmap.h>
#include <vm/vm_domainset.h>		#include <vm/vm_domainset.h>
#include <vm/vm_pageout.h>		#include <vm/vm_pageout.h>
#include <vm/vm_param.h>		#include <vm/vm_param.h>
#include <vm/vm_kern.h>		#include <vm/vm_kern.h>
#include <vm/vm_extern.h>		#include <vm/vm_extern.h>
#include <vm/vm_map.h>		#include <vm/vm_map.h>
#include <vm/vm_page.h>		#include <vm/vm_page.h>
		#include <vm/vm_phys.h>
		#include <vm/vm_pagequeue.h>
#include <vm/uma.h>		#include <vm/uma.h>
#include <vm/uma_int.h>		#include <vm/uma_int.h>
#include <vm/uma_dbg.h>		#include <vm/uma_dbg.h>

#ifdef DEBUG_MEMGUARD		#ifdef DEBUG_MEMGUARD
#include <vm/memguard.h>		#include <vm/memguard.h>
#endif		#endif
#ifdef DEBUG_REDZONE		#ifdef DEBUG_REDZONE
▲ Show 20 Lines • Show All 458 Lines • ▼ Show 20 Lines	#ifdef DEBUG_REDZONE
sizep = redzone_size_ntor(sizep);		sizep = redzone_size_ntor(sizep);
#endif		#endif

return (0);		return (0);
}		}
#endif		#endif

/*		/*
		* Handle large allocations and frees by using kmem_malloc directly.
		*/
		static inline bool
		malloc_large_slab(uma_slab_t slab)
		{
		uintptr_t va;

		va = (uintptr_t)slab;
		return ((va & 1) != 0);
		}

		static inline size_t
		malloc_large_size(uma_slab_t slab)
		{
		uintptr_t va;

		va = (uintptr_t)slab;
		return (va >> 1);
		}

		static caddr_t
		malloc_large(size_t size, struct domainset policy, int flags)
		{
		vm_offset_t va;
		size_t sz;

		sz = roundup(*size, PAGE_SIZE);
		va = kmem_malloc_domainset(policy, sz, flags);
		if (va != 0) {
		/* The low bit is unused for slab pointers. */
		vsetzoneslab(va, NULL, (void *)((sz << 1) \| 1));
		uma_total_inc(sz);
		*size = sz;
		}
		return ((caddr_t)va);
		}

		static void
		free_large(void *addr, size_t size)
		{

		kmem_free((vm_offset_t)addr, size);
		uma_total_dec(size);
		}

		/*
* malloc:		* malloc:
*		*
* Allocate a block of memory.		* Allocate a block of memory.
*		*
* If M_NOWAIT is set, this routine will not block and return NULL if		* If M_NOWAIT is set, this routine will not block and return NULL if
* the allocation fails.		* the allocation fails.
*/		*/
void *		void *
Show All 20 Lines
#ifdef MALLOC_PROFILE		#ifdef MALLOC_PROFILE
krequests[size >> KMEM_ZSHIFT]++;		krequests[size >> KMEM_ZSHIFT]++;
#endif		#endif
va = uma_zalloc(zone, flags);		va = uma_zalloc(zone, flags);
if (va != NULL)		if (va != NULL)
size = zone->uz_size;		size = zone->uz_size;
malloc_type_zone_allocated(mtp, va == NULL ? 0 : size, indx);		malloc_type_zone_allocated(mtp, va == NULL ? 0 : size, indx);
} else {		} else {
size = roundup(size, PAGE_SIZE);		va = malloc_large(&size, DOMAINSET_RR(), flags);
zone = NULL;
va = uma_large_malloc(size, flags);
malloc_type_allocated(mtp, va == NULL ? 0 : size);		malloc_type_allocated(mtp, va == NULL ? 0 : size);
}		}
if (flags & M_WAITOK)		if (flags & M_WAITOK)
KASSERT(va != NULL, ("malloc(M_WAITOK) returned NULL"));		KASSERT(va != NULL, ("malloc(M_WAITOK) returned NULL"));
else if (va == NULL)		else if (va == NULL)
t_malloc_fail = time_uptime;		t_malloc_fail = time_uptime;
#ifdef DEBUG_REDZONE		#ifdef DEBUG_REDZONE
if (va != NULL)		if (va != NULL)
va = redzone_setup(va, osize);		va = redzone_setup(va, osize);
#endif		#endif
return ((void *) va);		return ((void *) va);
}		}

static void *		static void *
malloc_domain(size_t size, struct malloc_type *mtp, int domain, int flags)		malloc_domain(size_t size, int indxp, struct malloc_type mtp, int domain,
		int flags)
{		{
int indx;		int indx;
caddr_t va;		caddr_t va;
uma_zone_t zone;		uma_zone_t zone;
#if defined(DEBUG_REDZONE)
unsigned long osize = size;
#endif

#ifdef MALLOC_DEBUG		KASSERT(size <= kmem_zmax && (flags & M_EXEC) == 0,
va = NULL;		("malloc_domain: Called with bad flag / size combination."));
if (malloc_dbg(&va, &size, mtp, flags) != 0)
return (va);
#endif
if (size <= kmem_zmax && (flags & M_EXEC) == 0) {
if (size & KMEM_ZMASK)		if (size & KMEM_ZMASK)
size = (size & ~KMEM_ZMASK) + KMEM_ZBASE;		size = (size & ~KMEM_ZMASK) + KMEM_ZBASE;
indx = kmemsize[size >> KMEM_ZSHIFT];		indx = kmemsize[size >> KMEM_ZSHIFT];
zone = kmemzones[indx].kz_zone[mtp_get_subzone(mtp)];		zone = kmemzones[indx].kz_zone[mtp_get_subzone(mtp)];
#ifdef MALLOC_PROFILE		#ifdef MALLOC_PROFILE
krequests[size >> KMEM_ZSHIFT]++;		krequests[size >> KMEM_ZSHIFT]++;
#endif		#endif
va = uma_zalloc_domain(zone, NULL, domain, flags);		va = uma_zalloc_domain(zone, NULL, domain, flags);
if (va != NULL)		if (va != NULL)
size = zone->uz_size;		size = zone->uz_size;
malloc_type_zone_allocated(mtp, va == NULL ? 0 : size, indx);		*indxp = indx;
} else {
size = roundup(size, PAGE_SIZE);
zone = NULL;
va = uma_large_malloc_domain(size, domain, flags);
malloc_type_allocated(mtp, va == NULL ? 0 : size);
}
if (flags & M_WAITOK)
KASSERT(va != NULL, ("malloc(M_WAITOK) returned NULL"));
else if (va == NULL)
t_malloc_fail = time_uptime;
#ifdef DEBUG_REDZONE
if (va != NULL)
va = redzone_setup(va, osize);
#endif
return ((void *) va);		return ((void *) va);
}		}

void *		void *
malloc_domainset(size_t size, struct malloc_type mtp, struct domainset ds,		malloc_domainset(size_t size, struct malloc_type mtp, struct domainset ds,
int flags)		int flags)
{		{
struct vm_domainset_iter di;		struct vm_domainset_iter di;
void *ret;		caddr_t ret;
int domain;		int domain;
		int indx;

		#if defined(DEBUG_REDZONE)
		unsigned long osize = size;
		#endif
		#ifdef MALLOC_DEBUG
		ret= NULL;
		if (malloc_dbg(&ret, &size, mtp, flags) != 0)
		return (ret);
		#endif
		if (size <= kmem_zmax && (flags & M_EXEC) == 0) {
vm_domainset_iter_policy_init(&di, ds, &domain, &flags);		vm_domainset_iter_policy_init(&di, ds, &domain, &flags);
do {		do {
ret = malloc_domain(size, mtp, domain, flags);		ret = malloc_domain(size, &indx, mtp, domain, flags);
if (ret != NULL)		} while (ret == NULL &&
break;		vm_domainset_iter_policy(&di, &domain) == 0);
} while (vm_domainset_iter_policy(&di, &domain) == 0);		malloc_type_zone_allocated(mtp, ret == NULL ? 0 : size, indx);
		} else {
		/* Policy is handled by kmem. */
		ret = malloc_large(&size, ds, flags);
		malloc_type_allocated(mtp, ret == NULL ? 0 : size);
		}

		if (flags & M_WAITOK)
		KASSERT(ret != NULL, ("malloc(M_WAITOK) returned NULL"));
		else if (ret == NULL)
		t_malloc_fail = time_uptime;
		#ifdef DEBUG_REDZONE
		if (ret != NULL)
		ret = redzone_setup(ret, osize);
		#endif
return (ret);		return (ret);
}		}

void *		void *
mallocarray(size_t nmemb, size_t size, struct malloc_type *type, int flags)		mallocarray(size_t nmemb, size_t size, struct malloc_type *type, int flags)
{		{

if (WOULD_OVERFLOW(nmemb, size))		if (WOULD_OVERFLOW(nmemb, size))
▲ Show 20 Lines • Show All 76 Lines • ▼ Show 20 Lines	#endif
if (addr == NULL)		if (addr == NULL)
return;		return;

vtozoneslab((vm_offset_t)addr & (~UMA_SLAB_MASK), &zone, &slab);		vtozoneslab((vm_offset_t)addr & (~UMA_SLAB_MASK), &zone, &slab);
if (slab == NULL)		if (slab == NULL)
panic("free: address %p(%p) has not been allocated.\n",		panic("free: address %p(%p) has not been allocated.\n",
addr, (void *)((u_long)addr & (~UMA_SLAB_MASK)));		addr, (void *)((u_long)addr & (~UMA_SLAB_MASK)));

if (!(slab->us_flags & UMA_SLAB_MALLOC)) {		if (__predict_true(!malloc_large_slab(slab))) {
size = zone->uz_size;		size = zone->uz_size;
#ifdef INVARIANTS		#ifdef INVARIANTS
free_save_type(addr, mtp, size);		free_save_type(addr, mtp, size);
#endif		#endif
uma_zfree_arg(zone, addr, slab);		uma_zfree_arg(zone, addr, slab);
} else {		} else {
size = slab->us_size;		size = malloc_large_size(slab);
uma_large_free(slab);		free_large(addr, size);
}		}
malloc_type_freed(mtp, size);		malloc_type_freed(mtp, size);
}		}

void		void
free_domain(void addr, struct malloc_type mtp)		free_domain(void addr, struct malloc_type mtp)
{		{
uma_zone_t zone;		uma_zone_t zone;
Show All 9 Lines	#endif
if (addr == NULL)		if (addr == NULL)
return;		return;

vtozoneslab((vm_offset_t)addr & (~UMA_SLAB_MASK), &zone, &slab);		vtozoneslab((vm_offset_t)addr & (~UMA_SLAB_MASK), &zone, &slab);
if (slab == NULL)		if (slab == NULL)
panic("free_domain: address %p(%p) has not been allocated.\n",		panic("free_domain: address %p(%p) has not been allocated.\n",
addr, (void *)((u_long)addr & (~UMA_SLAB_MASK)));		addr, (void *)((u_long)addr & (~UMA_SLAB_MASK)));

if (!(slab->us_flags & UMA_SLAB_MALLOC)) {		if (__predict_true(!malloc_large_slab(slab))) {
size = zone->uz_size;		size = zone->uz_size;
#ifdef INVARIANTS		#ifdef INVARIANTS
free_save_type(addr, mtp, size);		free_save_type(addr, mtp, size);
#endif		#endif
uma_zfree_domain(zone, addr, slab);		uma_zfree_domain(zone, addr, slab);
} else {		} else {
size = slab->us_size;		size = malloc_large_size(slab);
uma_large_free(slab);		free_large(addr, size);
}		}
malloc_type_freed(mtp, size);		malloc_type_freed(mtp, size);
}		}

/*		/*
* realloc: change the size of a memory block		* realloc: change the size of a memory block
*/		*/
void *		void *
Show All 30 Lines
#else		#else
vtozoneslab((vm_offset_t)addr & (~UMA_SLAB_MASK), &zone, &slab);		vtozoneslab((vm_offset_t)addr & (~UMA_SLAB_MASK), &zone, &slab);

/* Sanity check */		/* Sanity check */
KASSERT(slab != NULL,		KASSERT(slab != NULL,
("realloc: address %p out of range", (void *)addr));		("realloc: address %p out of range", (void *)addr));

/* Get the size of the original block */		/* Get the size of the original block */
if (!(slab->us_flags & UMA_SLAB_MALLOC))		if (!malloc_large_slab(slab))
alloc = zone->uz_size;		alloc = zone->uz_size;
else		else
alloc = slab->us_size;		alloc = malloc_large_size(slab);

/* Reuse the original block if appropriate */		/* Reuse the original block if appropriate */
if (size <= alloc		if (size <= alloc
&& (size > (alloc >> REALLOC_FRACTION) \|\| alloc == MINALLOCSIZE))		&& (size > (alloc >> REALLOC_FRACTION) \|\| alloc == MINALLOCSIZE))
return (addr);		return (addr);
#endif /* !DEBUG_REDZONE */		#endif /* !DEBUG_REDZONE */

/* Allocate a new, bigger (or smaller) block */		/* Allocate a new, bigger (or smaller) block */
▲ Show 20 Lines • Show All 526 Lines • Show Last 20 Lines