Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/vm_page.c
Show First 20 Lines • Show All 186 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* The cache page zone is initialized later since we need to be able to allocate | * The cache page zone is initialized later since we need to be able to allocate | ||||
* pages before UMA is fully initialized. | * pages before UMA is fully initialized. | ||||
*/ | */ | ||||
static void | static void | ||||
vm_page_init_cache_zones(void *dummy __unused) | vm_page_init_cache_zones(void *dummy __unused) | ||||
{ | { | ||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
int i; | struct vm_pgcache *pgcache; | ||||
int domain, pool; | |||||
for (i = 0; i < vm_ndomains; i++) { | for (domain = 0; domain < vm_ndomains; domain++) { | ||||
vmd = VM_DOMAIN(i); | vmd = VM_DOMAIN(domain); | ||||
/* | /* | ||||
* Don't allow the page cache to take up more than .25% of | * Don't allow the page caches to take up more than .25% of | ||||
* memory. | * memory. | ||||
*/ | */ | ||||
if (vmd->vmd_page_count / 400 < 256 * mp_ncpus) | if (vmd->vmd_page_count / 400 < 256 * mp_ncpus * VM_NFREEPOOL) | ||||
continue; | continue; | ||||
vmd->vmd_pgcache = uma_zcache_create("vm pgcache", | for (pool = 0; pool < VM_NFREEPOOL; pool++) { | ||||
pgcache = &vmd->vmd_pgcache[pool]; | |||||
pgcache->domain = domain; | |||||
pgcache->pool = pool; | |||||
pgcache->zone = uma_zcache_create("vm pgcache", | |||||
sizeof(struct vm_page), NULL, NULL, NULL, NULL, | sizeof(struct vm_page), NULL, NULL, NULL, NULL, | ||||
vm_page_import, vm_page_release, vmd, | vm_page_import, vm_page_release, pgcache, | ||||
UMA_ZONE_MAXBUCKET | UMA_ZONE_VM); | UMA_ZONE_MAXBUCKET | UMA_ZONE_VM); | ||||
(void )uma_zone_set_maxcache(vmd->vmd_pgcache, 0); | (void)uma_zone_set_maxcache(pgcache->zone, 0); | ||||
} | } | ||||
} | } | ||||
} | |||||
SYSINIT(vm_page2, SI_SUB_VM_CONF, SI_ORDER_ANY, vm_page_init_cache_zones, NULL); | SYSINIT(vm_page2, SI_SUB_VM_CONF, SI_ORDER_ANY, vm_page_init_cache_zones, NULL); | ||||
/* Make sure that u_long is at least 64 bits when PAGE_SIZE is 32K. */ | /* Make sure that u_long is at least 64 bits when PAGE_SIZE is 32K. */ | ||||
#if PAGE_SIZE == 32768 | #if PAGE_SIZE == 32768 | ||||
#ifdef CTASSERT | #ifdef CTASSERT | ||||
CTASSERT(sizeof(u_long) >= 8); | CTASSERT(sizeof(u_long) >= 8); | ||||
#endif | #endif | ||||
#endif | #endif | ||||
▲ Show 20 Lines • Show All 1,572 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
vm_page_t | vm_page_t | ||||
vm_page_alloc_domain_after(vm_object_t object, vm_pindex_t pindex, int domain, | vm_page_alloc_domain_after(vm_object_t object, vm_pindex_t pindex, int domain, | ||||
int req, vm_page_t mpred) | int req, vm_page_t mpred) | ||||
{ | { | ||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
vm_page_t m; | vm_page_t m; | ||||
int flags; | int flags, pool; | ||||
KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) && | KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) && | ||||
(object != NULL || (req & VM_ALLOC_SBUSY) == 0) && | (object != NULL || (req & VM_ALLOC_SBUSY) == 0) && | ||||
((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) != | ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) != | ||||
(VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), | (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)), | ||||
("inconsistent object(%p)/req(%x)", object, req)); | ("inconsistent object(%p)/req(%x)", object, req)); | ||||
KASSERT(object == NULL || (req & VM_ALLOC_WAITOK) == 0, | KASSERT(object == NULL || (req & VM_ALLOC_WAITOK) == 0, | ||||
("Can't sleep and retry object insertion.")); | ("Can't sleep and retry object insertion.")); | ||||
KASSERT(mpred == NULL || mpred->pindex < pindex, | KASSERT(mpred == NULL || mpred->pindex < pindex, | ||||
("mpred %p doesn't precede pindex 0x%jx", mpred, | ("mpred %p doesn't precede pindex 0x%jx", mpred, | ||||
(uintmax_t)pindex)); | (uintmax_t)pindex)); | ||||
if (object != NULL) | if (object != NULL) | ||||
VM_OBJECT_ASSERT_WLOCKED(object); | VM_OBJECT_ASSERT_WLOCKED(object); | ||||
flags = 0; | flags = 0; | ||||
m = NULL; | m = NULL; | ||||
pool = object != NULL ? VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT; | |||||
again: | again: | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
/* | /* | ||||
* Can we allocate the page from a reservation? | * Can we allocate the page from a reservation? | ||||
*/ | */ | ||||
if (vm_object_reserv(object) && | if (vm_object_reserv(object) && | ||||
(m = vm_reserv_alloc_page(object, pindex, domain, req, mpred)) != | (m = vm_reserv_alloc_page(object, pindex, domain, req, mpred)) != | ||||
NULL) { | NULL) { | ||||
domain = vm_phys_domain(m); | domain = vm_phys_domain(m); | ||||
vmd = VM_DOMAIN(domain); | vmd = VM_DOMAIN(domain); | ||||
goto found; | goto found; | ||||
} | } | ||||
#endif | #endif | ||||
vmd = VM_DOMAIN(domain); | vmd = VM_DOMAIN(domain); | ||||
if (object != NULL && vmd->vmd_pgcache != NULL) { | if (vmd->vmd_pgcache[pool].zone != NULL) { | ||||
m = uma_zalloc(vmd->vmd_pgcache, M_NOWAIT); | m = uma_zalloc(vmd->vmd_pgcache[pool].zone, M_NOWAIT); | ||||
if (m != NULL) { | if (m != NULL) { | ||||
flags |= PG_PCPU_CACHE; | flags |= PG_PCPU_CACHE; | ||||
goto found; | goto found; | ||||
} | } | ||||
} | } | ||||
if (vm_domain_allocate(vmd, req, 1)) { | if (vm_domain_allocate(vmd, req, 1)) { | ||||
/* | /* | ||||
* If not, allocate it from the free page queues. | * If not, allocate it from the free page queues. | ||||
*/ | */ | ||||
vm_domain_free_lock(vmd); | vm_domain_free_lock(vmd); | ||||
m = vm_phys_alloc_pages(domain, object != NULL ? | m = vm_phys_alloc_pages(domain, pool, 0); | ||||
VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, 0); | |||||
vm_domain_free_unlock(vmd); | vm_domain_free_unlock(vmd); | ||||
if (m == NULL) { | if (m == NULL) { | ||||
vm_domain_freecnt_inc(vmd, 1); | vm_domain_freecnt_inc(vmd, 1); | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
if (vm_reserv_reclaim_inactive(domain)) | if (vm_reserv_reclaim_inactive(domain)) | ||||
goto again; | goto again; | ||||
#endif | #endif | ||||
} | } | ||||
▲ Show 20 Lines • Show All 373 Lines • ▼ Show 20 Lines | again: | ||||
m->oflags = VPO_UNMANAGED; | m->oflags = VPO_UNMANAGED; | ||||
return (m); | return (m); | ||||
} | } | ||||
static int | static int | ||||
vm_page_import(void *arg, void **store, int cnt, int domain, int flags) | vm_page_import(void *arg, void **store, int cnt, int domain, int flags) | ||||
{ | { | ||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
struct vm_pgcache *pgcache; | |||||
int i; | int i; | ||||
vmd = arg; | pgcache = arg; | ||||
vmd = VM_DOMAIN(pgcache->domain); | |||||
/* Only import if we can bring in a full bucket. */ | /* Only import if we can bring in a full bucket. */ | ||||
if (cnt == 1 || !vm_domain_allocate(vmd, VM_ALLOC_NORMAL, cnt)) | if (cnt == 1 || !vm_domain_allocate(vmd, VM_ALLOC_NORMAL, cnt)) | ||||
return (0); | return (0); | ||||
domain = vmd->vmd_domain; | domain = vmd->vmd_domain; | ||||
vm_domain_free_lock(vmd); | vm_domain_free_lock(vmd); | ||||
i = vm_phys_alloc_npages(domain, VM_FREEPOOL_DEFAULT, cnt, | i = vm_phys_alloc_npages(domain, pgcache->pool, cnt, | ||||
(vm_page_t *)store); | (vm_page_t *)store); | ||||
vm_domain_free_unlock(vmd); | vm_domain_free_unlock(vmd); | ||||
if (cnt != i) | if (cnt != i) | ||||
vm_domain_freecnt_inc(vmd, cnt - i); | vm_domain_freecnt_inc(vmd, cnt - i); | ||||
return (i); | return (i); | ||||
} | } | ||||
static void | static void | ||||
vm_page_release(void *arg, void **store, int cnt) | vm_page_release(void *arg, void **store, int cnt) | ||||
{ | { | ||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
struct vm_pgcache *pgcache; | |||||
vm_page_t m; | vm_page_t m; | ||||
int i; | int i; | ||||
vmd = arg; | pgcache = arg; | ||||
vmd = VM_DOMAIN(pgcache->domain); | |||||
vm_domain_free_lock(vmd); | vm_domain_free_lock(vmd); | ||||
for (i = 0; i < cnt; i++) { | for (i = 0; i < cnt; i++) { | ||||
m = (vm_page_t)store[i]; | m = (vm_page_t)store[i]; | ||||
vm_phys_free_pages(m, 0); | vm_phys_free_pages(m, 0); | ||||
} | } | ||||
vm_domain_free_unlock(vmd); | vm_domain_free_unlock(vmd); | ||||
vm_domain_freecnt_inc(vmd, cnt); | vm_domain_freecnt_inc(vmd, cnt); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 1,228 Lines • ▼ Show 20 Lines | |||||
* | * | ||||
* The object must be locked. The page must be locked if it is | * The object must be locked. The page must be locked if it is | ||||
* managed. | * managed. | ||||
*/ | */ | ||||
void | void | ||||
vm_page_free_toq(vm_page_t m) | vm_page_free_toq(vm_page_t m) | ||||
{ | { | ||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
uma_zone_t zone; | |||||
if (!vm_page_free_prep(m)) | if (!vm_page_free_prep(m)) | ||||
return; | return; | ||||
vmd = vm_pagequeue_domain(m); | vmd = vm_pagequeue_domain(m); | ||||
if ((m->flags & PG_PCPU_CACHE) != 0 && vmd->vmd_pgcache != NULL) { | zone = vmd->vmd_pgcache[m->pool].zone; | ||||
uma_zfree(vmd->vmd_pgcache, m); | if ((m->flags & PG_PCPU_CACHE) != 0 && zone != NULL) { | ||||
uma_zfree(zone, m); | |||||
return; | return; | ||||
} | } | ||||
vm_domain_free_lock(vmd); | vm_domain_free_lock(vmd); | ||||
vm_phys_free_pages(m, 0); | vm_phys_free_pages(m, 0); | ||||
vm_domain_free_unlock(vmd); | vm_domain_free_unlock(vmd); | ||||
vm_domain_freecnt_inc(vmd, 1); | vm_domain_freecnt_inc(vmd, 1); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 1,013 Lines • Show Last 20 Lines |