Index: share/man/man9/zone.9 =================================================================== --- share/man/man9/zone.9 +++ share/man/man9/zone.9 @@ -340,6 +340,10 @@ See .Xr numa 4 for more details. +.It Dv UMA_ZONE_CONTIG +Items in this zone must be contiguous in physical address space. Items will +follow normal alignment constraints and may span page boundaries, but only +between pages with contiguous physical addresses. .El .Pp Zones can be destroyed using @@ -466,11 +470,11 @@ functions allow a zone's default slab allocation and free functions to be overridden. This is useful if the zone's items have special memory allocation constraints. -For example, if multi-page objects are required to be physically contiguous, -an +For example, if memory with a memory attribute other than +.Dv VM_MEMATTR_DEFAULT +must be used, then an .Fa allocf -function which requests contiguous memory from the kernel's page allocator -may be used. +function which yields such memory should be set. .Pp The .Fn uma_zone_set_max Index: sys/vm/uma.h =================================================================== --- sys/vm/uma.h +++ sys/vm/uma.h @@ -235,6 +235,7 @@ * overlap when adding new features. */ #define UMA_ZONE_ZINIT 0x0002 /* Initialize with zeros */ +#define UMA_ZONE_CONTIG 0x0004 /* Objects are physically contiguous */ #define UMA_ZONE_NOTOUCH 0x0008 /* UMA may not access the memory */ #define UMA_ZONE_MALLOC 0x0010 /* For use by malloc(9) only! */ #define UMA_ZONE_NOFREE 0x0020 /* Do not free slabs of this type! */ @@ -606,8 +607,7 @@ #define UMA_SLAB_BOOT 0x01 /* Slab alloced from boot pages */ #define UMA_SLAB_KERNEL 0x04 /* Slab alloced from kmem */ #define UMA_SLAB_PRIV 0x08 /* Slab alloced from priv allocator */ -#define UMA_SLAB_OFFP 0x10 /* Slab is managed separately */ -/* 0x02, 0x40, and 0x80 are available */ +/* 0x02, 0x10, 0x40, and 0x80 are available */ /* * Used to pre-fill a zone with some number of items Index: sys/vm/uma_core.c =================================================================== --- sys/vm/uma_core.c +++ sys/vm/uma_core.c @@ -263,6 +263,7 @@ static void *page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); static void *pcpu_page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); static void *startup_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); +static void *contig_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); static void page_free(void *, vm_size_t, uint8_t); static void pcpu_page_free(void *, vm_size_t, uint8_t); static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int, int, int); @@ -1607,6 +1608,19 @@ return ((void *)retkva); } +/* + * Allocate physically contiguous pages. + */ +static void * +contig_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, + int wait) +{ + + *pflag = UMA_SLAB_KERNEL; + return ((void *)kmem_alloc_contig_domainset(DOMAINSET_FIXED(domain), + bytes, wait, 0, ~(vm_paddr_t)0, 1, 0, VM_MEMATTR_DEFAULT)); +} + /* * Frees a number of pages to the system * @@ -1627,8 +1641,8 @@ return; } - if ((flags & UMA_SLAB_KERNEL) == 0) - panic("UMA: page_free used with invalid flags %x", flags); + KASSERT((flags & UMA_SLAB_KERNEL) != 0, + ("UMA: page_free used with invalid flags %x", flags)); kmem_free((vm_offset_t)mem, size); } @@ -1952,6 +1966,9 @@ if (arg->flags & UMA_ZONE_MALLOC) keg->uk_flags |= UMA_ZFLAG_VTOSLAB; + KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 || booted >= BOOT_KVA, + ("Early boot PCPU zones not implemented, keg %s flags 0x%#b", + keg->uk_name, keg->uk_flags, PRINT_UMA_ZFLAGS)); #ifndef SMP keg->uk_flags &= ~UMA_ZONE_PCPU; #endif @@ -1986,6 +2003,8 @@ keg->uk_allocf = startup_alloc; else if (keg->uk_flags & UMA_ZONE_PCPU) keg->uk_allocf = pcpu_page_alloc; + else if ((keg->uk_flags & UMA_ZONE_CONTIG) != 0 && keg->uk_ppera > 1) + keg->uk_allocf = contig_alloc; else keg->uk_allocf = page_alloc; #ifdef UMA_MD_SMALL_ALLOC @@ -2047,8 +2066,13 @@ if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0) return; KEG_GET(zone, keg); - if (keg->uk_allocf == startup_alloc) - keg->uk_allocf = page_alloc; + if (keg->uk_allocf == startup_alloc) { + if ((keg->uk_flags & UMA_ZONE_CONTIG) != 0 && + keg->uk_ppera > 1) + keg->uk_allocf = contig_alloc; + else + keg->uk_allocf = page_alloc; + } } static void Index: sys/vm/uma_int.h =================================================================== --- sys/vm/uma_int.h +++ sys/vm/uma_int.h @@ -199,6 +199,7 @@ "\6NOFREE" \ "\5MALLOC" \ "\4NOTOUCH" \ + "\3CONTIG" \ "\2ZINIT" /*