Changeset View
Standalone View
sys/vm/vm_page.c
Show First 20 Lines • Show All 1,797 Lines • ▼ Show 20 Lines | vm_page_alloc_domain_after(vm_object_t object, vm_pindex_t pindex, int domain, | ||||
KASSERT(object == NULL || (req & VM_ALLOC_WAITOK) == 0, | KASSERT(object == NULL || (req & VM_ALLOC_WAITOK) == 0, | ||||
("Can't sleep and retry object insertion.")); | ("Can't sleep and retry object insertion.")); | ||||
KASSERT(mpred == NULL || mpred->pindex < pindex, | KASSERT(mpred == NULL || mpred->pindex < pindex, | ||||
("mpred %p doesn't precede pindex 0x%jx", mpred, | ("mpred %p doesn't precede pindex 0x%jx", mpred, | ||||
(uintmax_t)pindex)); | (uintmax_t)pindex)); | ||||
if (object != NULL) | if (object != NULL) | ||||
VM_OBJECT_ASSERT_WLOCKED(object); | VM_OBJECT_ASSERT_WLOCKED(object); | ||||
again: | flags = 0; | ||||
dougm: The optimization of moving m = NULL to before 'again' is also available with 'm_ret' and… | |||||
m = NULL; | m = NULL; | ||||
again: | |||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
/* | /* | ||||
* Can we allocate the page from a reservation? | * Can we allocate the page from a reservation? | ||||
*/ | */ | ||||
if (vm_object_reserv(object) && | if (vm_object_reserv(object) && | ||||
(m = vm_reserv_alloc_page(object, pindex, domain, req, mpred)) != | (m = vm_reserv_alloc_page(object, pindex, domain, req, mpred)) != | ||||
NULL) { | NULL) { | ||||
domain = vm_phys_domain(m); | domain = vm_phys_domain(m); | ||||
vmd = VM_DOMAIN(domain); | vmd = VM_DOMAIN(domain); | ||||
goto found; | goto found; | ||||
} | } | ||||
#endif | #endif | ||||
vmd = VM_DOMAIN(domain); | vmd = VM_DOMAIN(domain); | ||||
if (object != NULL && vmd->vmd_pgcache != NULL) { | if (object != NULL && vmd->vmd_pgcache != NULL) { | ||||
m = uma_zalloc(vmd->vmd_pgcache, M_NOWAIT); | m = uma_zalloc(vmd->vmd_pgcache, M_NOWAIT); | ||||
if (m != NULL) | if (m != NULL) { | ||||
flags |= PG_CACHE_ALLOC; | |||||
goto found; | goto found; | ||||
} | } | ||||
} | |||||
if (vm_domain_allocate(vmd, req, 1)) { | if (vm_domain_allocate(vmd, req, 1)) { | ||||
/* | /* | ||||
* If not, allocate it from the free page queues. | * If not, allocate it from the free page queues. | ||||
*/ | */ | ||||
vm_domain_free_lock(vmd); | vm_domain_free_lock(vmd); | ||||
m = vm_phys_alloc_pages(domain, object != NULL ? | m = vm_phys_alloc_pages(domain, object != NULL ? | ||||
VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, 0); | VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, 0); | ||||
vm_domain_free_unlock(vmd); | vm_domain_free_unlock(vmd); | ||||
Show All 9 Lines | if (m == NULL) { | ||||
/* | /* | ||||
* Not allocatable, give up. | * Not allocatable, give up. | ||||
*/ | */ | ||||
if (vm_domain_alloc_fail(vmd, object, req)) | if (vm_domain_alloc_fail(vmd, object, req)) | ||||
goto again; | goto again; | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
/* | |||||
* At this point we had better have found a good page. | |||||
*/ | |||||
alcUnsubmitted Done Inline ActionsI might keep this comment, but I agree with deleting the KASSERT. alc: I might keep this comment, but I agree with deleting the KASSERT. | |||||
KASSERT(m != NULL, ("missing page")); | |||||
found: | found: | ||||
vm_page_dequeue(m); | vm_page_dequeue(m); | ||||
vm_page_alloc_check(m); | vm_page_alloc_check(m); | ||||
/* | /* | ||||
* Initialize the page. Only the PG_ZERO flag is inherited. | * Initialize the page. The PG_ZERO flag is inherited. | ||||
*/ | */ | ||||
flags = 0; | |||||
Not Done Inline ActionsIf you initialize flags to PG_CACHE_ALLOC here ... alc: If you initialize flags to PG_CACHE_ALLOC here ... | |||||
if ((req & VM_ALLOC_ZERO) != 0) | if ((req & VM_ALLOC_ZERO) != 0) | ||||
flags = PG_ZERO; | flags |= (m->flags & PG_ZERO); | ||||
Not Done Inline Actionsand change this to flags |= PG_ZERO, then PG_CACHE_ALLOC would be "inherited" like PG_ZERO. And, so you would only set PG_CACHE_ALLOC in vm_phys_alloc_npages() and clear it in vm_page_release(). IMO, the downside to this suggested approach is that setting PG_CACHE_ALLOC in vm_phys_alloc_npages() feels like we would be blurring the abstraction boundary between vm_phys and vm_page. Thoughts? alc: and change this to flags |= PG_ZERO, then PG_CACHE_ALLOC would be "inherited" like PG_ZERO. | |||||
Done Inline ActionsIndeed, I thought about setting it in vm_phys_alloc_npages() and decided against it because it felt like a layering violation. (And vm_phys_alloc_npages() could conceivably be used for non-pcpu cache allocations, for example in the vm_page_alloc_pages() function I wrote a while ago.) If we set m->flags |= PG_CACHE_ALLOC after allocating from UMA and clear it in vm_page_release() we could do as you suggested above: flags = PG_CACHE_ALLOC; if ((req & VM_ALLOC_ZERO) != 0) flags |= PG_ZERO; flags &= m->flags; if ((req & VM_ALLOC_NODUMP) != 0) flags |= PG_NODUMP; m->flags = flags; There would be some asymmetry in that we set PG_CACHE_ALLOC after allocating from the cache and clear it only when releasing pages from the cache, but to me that seems preferable to setting PG_CACHE_ALLOC in vm_phys. markj: Indeed, I thought about setting it in vm_phys_alloc_npages() and decided against it because it… | |||||
flags &= m->flags; | |||||
if ((req & VM_ALLOC_NODUMP) != 0) | if ((req & VM_ALLOC_NODUMP) != 0) | ||||
flags |= PG_NODUMP; | flags |= PG_NODUMP; | ||||
m->flags = flags; | m->flags = flags; | ||||
m->aflags = 0; | m->aflags = 0; | ||||
m->oflags = object == NULL || (object->flags & OBJ_UNMANAGED) != 0 ? | m->oflags = object == NULL || (object->flags & OBJ_UNMANAGED) != 0 ? | ||||
VPO_UNMANAGED : 0; | VPO_UNMANAGED : 0; | ||||
m->busy_lock = VPB_UNBUSIED; | m->busy_lock = VPB_UNBUSIED; | ||||
if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0) | if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0) | ||||
▲ Show 20 Lines • Show All 129 Lines • ▼ Show 20 Lines | if (object != NULL) { | ||||
KASSERT(mpred == NULL || mpred->pindex != pindex, | KASSERT(mpred == NULL || mpred->pindex != pindex, | ||||
("vm_page_alloc_contig: pindex already allocated")); | ("vm_page_alloc_contig: pindex already allocated")); | ||||
} | } | ||||
/* | /* | ||||
* Can we allocate the pages without the number of free pages falling | * Can we allocate the pages without the number of free pages falling | ||||
* below the lower bound for the allocation class? | * below the lower bound for the allocation class? | ||||
*/ | */ | ||||
m_ret = NULL; | |||||
again: | again: | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
/* | /* | ||||
* Can we allocate the pages from a reservation? | * Can we allocate the pages from a reservation? | ||||
*/ | */ | ||||
if (vm_object_reserv(object) && | if (vm_object_reserv(object) && | ||||
(m_ret = vm_reserv_alloc_contig(object, pindex, domain, req, | (m_ret = vm_reserv_alloc_contig(object, pindex, domain, req, | ||||
mpred, npages, low, high, alignment, boundary)) != NULL) { | mpred, npages, low, high, alignment, boundary)) != NULL) { | ||||
domain = vm_phys_domain(m_ret); | domain = vm_phys_domain(m_ret); | ||||
vmd = VM_DOMAIN(domain); | vmd = VM_DOMAIN(domain); | ||||
goto found; | goto found; | ||||
} | } | ||||
#endif | #endif | ||||
m_ret = NULL; | |||||
vmd = VM_DOMAIN(domain); | vmd = VM_DOMAIN(domain); | ||||
if (vm_domain_allocate(vmd, req, npages)) { | if (vm_domain_allocate(vmd, req, npages)) { | ||||
/* | /* | ||||
* allocate them from the free page queues. | * allocate them from the free page queues. | ||||
*/ | */ | ||||
vm_domain_free_lock(vmd); | vm_domain_free_lock(vmd); | ||||
m_ret = vm_phys_alloc_contig(domain, npages, low, high, | m_ret = vm_phys_alloc_contig(domain, npages, low, high, | ||||
alignment, boundary); | alignment, boundary); | ||||
▲ Show 20 Lines • Show All 1,447 Lines • ▼ Show 20 Lines | |||||
vm_page_free_toq(vm_page_t m) | vm_page_free_toq(vm_page_t m) | ||||
{ | { | ||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
if (!vm_page_free_prep(m)) | if (!vm_page_free_prep(m)) | ||||
return; | return; | ||||
vmd = vm_pagequeue_domain(m); | vmd = vm_pagequeue_domain(m); | ||||
if (m->pool == VM_FREEPOOL_DEFAULT && vmd->vmd_pgcache != NULL) { | if ((m->flags & PG_CACHE_ALLOC) != 0 && vmd->vmd_pgcache != NULL) { | ||||
Done Inline ActionsI think we could also use this flag to avoid the vm_reserv_free_page() call in vm_page_free_prep(). As I noted in the email thread, the access of the reservation structure usually results in a cache miss. markj: I think we could also use this flag to avoid the vm_reserv_free_page() call in… | |||||
Done Inline Actions... this would still not be optimal, e.g., since pages allocated with VM_ALLOC_NOOBJ never come from the per-CPU caches but will also never be allocated from a reservation. markj: ... this would still not be optimal, e.g., since pages allocated with VM_ALLOC_NOOBJ never come… | |||||
Not Done Inline ActionsIs there any reason that we don't have a per-pool vmd_cache? alc: Is there any reason that we don't have a per-pool vmd_cache? | |||||
Done Inline ActionsI can't think of one. When I first implemented the UMA-based page cache I indeed had a zone per freepool. I just asked Drew about it, and it turns out that Netflix still a version with one zone per freepool. It's necessary for their in-kernel TLS implementation, which does VM_ALLOC_NOOBJ allocations in the encrypted sendfile path. Given this information, I would propose bringing that extension (one zone per freepool) to FreeBSD. Then we can use this flag or some variant to elide vm_reserv_free_page() calls for many NOOBJ allocations as well. markj: I can't think of one. When I first implemented the UMA-based page cache I indeed had a zone per… | |||||
uma_zfree(vmd->vmd_pgcache, m); | uma_zfree(vmd->vmd_pgcache, m); | ||||
return; | return; | ||||
} | } | ||||
vm_domain_free_lock(vmd); | vm_domain_free_lock(vmd); | ||||
vm_phys_free_pages(m, 0); | vm_phys_free_pages(m, 0); | ||||
vm_domain_free_unlock(vmd); | vm_domain_free_unlock(vmd); | ||||
vm_domain_freecnt_inc(vmd, 1); | vm_domain_freecnt_inc(vmd, 1); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 1,159 Lines • Show Last 20 Lines |
The optimization of moving m = NULL to before 'again' is also available with 'm_ret' and 'again' in vm_page_alloc_contig_domain.