Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/swap_pager.c
Show First 20 Lines • Show All 93 Lines • ▼ Show 20 Lines | |||||
#include <sys/resource.h> | #include <sys/resource.h> | ||||
#include <sys/resourcevar.h> | #include <sys/resourcevar.h> | ||||
#include <sys/rwlock.h> | #include <sys/rwlock.h> | ||||
#include <sys/sbuf.h> | #include <sys/sbuf.h> | ||||
#include <sys/sysctl.h> | #include <sys/sysctl.h> | ||||
#include <sys/sysproto.h> | #include <sys/sysproto.h> | ||||
#include <sys/systm.h> | #include <sys/systm.h> | ||||
#include <sys/sx.h> | #include <sys/sx.h> | ||||
#include <sys/user.h> | |||||
#include <sys/vmmeter.h> | #include <sys/vmmeter.h> | ||||
#include <sys/vnode.h> | #include <sys/vnode.h> | ||||
#include <security/mac/mac_framework.h> | #include <security/mac/mac_framework.h> | ||||
#include <vm/vm.h> | #include <vm/vm.h> | ||||
#include <vm/pmap.h> | #include <vm/pmap.h> | ||||
#include <vm/vm_map.h> | #include <vm/vm_map.h> | ||||
▲ Show 20 Lines • Show All 303 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* pagerops for OBJT_SWAP - "swap pager". Some ops are also global procedure | * pagerops for OBJT_SWAP - "swap pager". Some ops are also global procedure | ||||
* calls hooked from other parts of the VM system and do not appear here. | * calls hooked from other parts of the VM system and do not appear here. | ||||
* (see vm/swap_pager.h). | * (see vm/swap_pager.h). | ||||
*/ | */ | ||||
static vm_object_t | static vm_object_t | ||||
swap_pager_alloc(void *handle, vm_ooffset_t size, | swap_pager_alloc(void *handle, vm_ooffset_t size, | ||||
vm_prot_t prot, vm_ooffset_t offset, struct ucred *); | vm_prot_t prot, vm_ooffset_t offset, struct ucred *); | ||||
static vm_object_t | |||||
swap_tmpfs_pager_alloc(void *handle, vm_ooffset_t size, | |||||
vm_prot_t prot, vm_ooffset_t offset, struct ucred *); | |||||
static void swap_pager_dealloc(vm_object_t object); | static void swap_pager_dealloc(vm_object_t object); | ||||
static int swap_pager_getpages(vm_object_t, vm_page_t *, int, int *, | static int swap_pager_getpages(vm_object_t, vm_page_t *, int, int *, | ||||
int *); | int *); | ||||
static int swap_pager_getpages_async(vm_object_t, vm_page_t *, int, int *, | static int swap_pager_getpages_async(vm_object_t, vm_page_t *, int, int *, | ||||
int *, pgo_getpages_iodone_t, void *); | int *, pgo_getpages_iodone_t, void *); | ||||
static void swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *); | static void swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *); | ||||
static boolean_t | static boolean_t | ||||
swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after); | swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after); | ||||
static void swap_pager_init(void); | static void swap_pager_init(void); | ||||
static void swap_pager_unswapped(vm_page_t); | static void swap_pager_unswapped(vm_page_t); | ||||
static void swap_pager_swapoff(struct swdevt *sp); | static void swap_pager_swapoff(struct swdevt *sp); | ||||
static void swap_pager_update_writecount(vm_object_t object, | static void swap_pager_update_writecount(vm_object_t object, | ||||
vm_offset_t start, vm_offset_t end); | vm_offset_t start, vm_offset_t end); | ||||
static void swap_pager_release_writecount(vm_object_t object, | static void swap_pager_release_writecount(vm_object_t object, | ||||
vm_offset_t start, vm_offset_t end); | vm_offset_t start, vm_offset_t end); | ||||
static void swap_tmpfs_pager_getvp(vm_object_t object, struct vnode **vpp, | |||||
bool *vp_heldp); | |||||
static void swap_pager_freespace(vm_object_t object, vm_pindex_t start, | static void swap_pager_freespace(vm_object_t object, vm_pindex_t start, | ||||
vm_size_t size); | vm_size_t size); | ||||
const struct pagerops swappagerops = { | const struct pagerops swappagerops = { | ||||
.pgo_kvme_type = KVME_TYPE_SWAP, | |||||
.pgo_init = swap_pager_init, /* early system initialization of pager */ | .pgo_init = swap_pager_init, /* early system initialization of pager */ | ||||
.pgo_alloc = swap_pager_alloc, /* allocate an OBJT_SWAP object */ | .pgo_alloc = swap_pager_alloc, /* allocate an OBJT_SWAP object */ | ||||
.pgo_dealloc = swap_pager_dealloc, /* deallocate an OBJT_SWAP object */ | .pgo_dealloc = swap_pager_dealloc, /* deallocate an OBJT_SWAP object */ | ||||
.pgo_getpages = swap_pager_getpages, /* pagein */ | .pgo_getpages = swap_pager_getpages, /* pagein */ | ||||
.pgo_getpages_async = swap_pager_getpages_async, /* pagein (async) */ | .pgo_getpages_async = swap_pager_getpages_async, /* pagein (async) */ | ||||
.pgo_putpages = swap_pager_putpages, /* pageout */ | .pgo_putpages = swap_pager_putpages, /* pageout */ | ||||
.pgo_haspage = swap_pager_haspage, /* get backing store status for page */ | .pgo_haspage = swap_pager_haspage, /* get backing store status for page */ | ||||
.pgo_pageunswapped = swap_pager_unswapped, /* remove swap related to page */ | .pgo_pageunswapped = swap_pager_unswapped, /* remove swap related to page */ | ||||
.pgo_update_writecount = swap_pager_update_writecount, | .pgo_update_writecount = swap_pager_update_writecount, | ||||
.pgo_release_writecount = swap_pager_release_writecount, | .pgo_release_writecount = swap_pager_release_writecount, | ||||
.pgo_freespace = swap_pager_freespace, | .pgo_freespace = swap_pager_freespace, | ||||
}; | }; | ||||
const struct pagerops swaptmpfspagerops = { | |||||
.pgo_alloc = swap_tmpfs_pager_alloc, | |||||
.pgo_dealloc = swap_pager_dealloc, | |||||
.pgo_getpages = swap_pager_getpages, | |||||
.pgo_getpages_async = swap_pager_getpages_async, | |||||
.pgo_putpages = swap_pager_putpages, | |||||
.pgo_haspage = swap_pager_haspage, | |||||
.pgo_pageunswapped = swap_pager_unswapped, | |||||
.pgo_update_writecount = swap_pager_update_writecount, | |||||
.pgo_release_writecount = swap_pager_release_writecount, | |||||
.pgo_set_writeable_dirty = vm_object_set_writeable_dirty_, | |||||
.pgo_mightbedirty = vm_object_mightbedirty_, | |||||
.pgo_getvp = swap_tmpfs_pager_getvp, | |||||
.pgo_freespace = swap_pager_freespace, | |||||
}; | |||||
/* | /* | ||||
* swap_*() routines are externally accessible. swp_*() routines are | * swap_*() routines are externally accessible. swp_*() routines are | ||||
* internal. | * internal. | ||||
*/ | */ | ||||
static int nswap_lowat = 128; /* in pages, swap_pager_almost_full warn */ | static int nswap_lowat = 128; /* in pages, swap_pager_almost_full warn */ | ||||
static int nswap_hiwat = 512; /* in pages, swap_pager_almost_full warn */ | static int nswap_hiwat = 512; /* in pages, swap_pager_almost_full warn */ | ||||
SYSCTL_INT(_vm, OID_AUTO, dmmax, CTLFLAG_RD, &nsw_cluster_max, 0, | SYSCTL_INT(_vm, OID_AUTO, dmmax, CTLFLAG_RD, &nsw_cluster_max, 0, | ||||
▲ Show 20 Lines • Show All 191 Lines • ▼ Show 20 Lines | swap_pager_swap_init(void) | ||||
/* absolute maximum we can handle assuming 100% efficiency */ | /* absolute maximum we can handle assuming 100% efficiency */ | ||||
swap_maxpages = n * SWAP_META_PAGES; | swap_maxpages = n * SWAP_META_PAGES; | ||||
swzone = n * sizeof(struct swblk); | swzone = n * sizeof(struct swblk); | ||||
if (!uma_zone_reserve_kva(swpctrie_zone, n)) | if (!uma_zone_reserve_kva(swpctrie_zone, n)) | ||||
printf("Cannot reserve swap pctrie zone, " | printf("Cannot reserve swap pctrie zone, " | ||||
"reduce kern.maxswzone.\n"); | "reduce kern.maxswzone.\n"); | ||||
} | } | ||||
static vm_object_t | bool | ||||
swap_pager_alloc_init(objtype_t otype, void *handle, struct ucred *cred, | swap_pager_init_object(vm_object_t object, void *handle, struct ucred *cred, | ||||
vm_ooffset_t size, vm_ooffset_t offset) | vm_ooffset_t size, vm_ooffset_t offset) | ||||
{ | { | ||||
vm_object_t object; | |||||
if (cred != NULL) { | if (cred != NULL) { | ||||
if (!swap_reserve_by_cred(size, cred)) | if (!swap_reserve_by_cred(size, cred)) | ||||
return (NULL); | return (false); | ||||
crhold(cred); | crhold(cred); | ||||
} | } | ||||
object->un_pager.swp.writemappings = 0; | |||||
object->handle = handle; | |||||
if (cred != NULL) { | |||||
object->cred = cred; | |||||
object->charge = size; | |||||
} | |||||
return (true); | |||||
} | |||||
static vm_object_t | |||||
swap_pager_alloc_init(objtype_t otype, void *handle, struct ucred *cred, | |||||
vm_ooffset_t size, vm_ooffset_t offset) | |||||
{ | |||||
vm_object_t object; | |||||
/* | /* | ||||
* The un_pager.swp.swp_blks trie is initialized by | * The un_pager.swp.swp_blks trie is initialized by | ||||
* vm_object_allocate() to ensure the correct order of | * vm_object_allocate() to ensure the correct order of | ||||
* visibility to other threads. | * visibility to other threads. | ||||
*/ | */ | ||||
object = vm_object_allocate(otype, OFF_TO_IDX(offset + | object = vm_object_allocate(otype, OFF_TO_IDX(offset + | ||||
PAGE_MASK + size)); | PAGE_MASK + size)); | ||||
object->un_pager.swp.writemappings = 0; | if (!swap_pager_init_object(object, handle, cred, size, offset)) { | ||||
object->handle = handle; | vm_object_deallocate(object); | ||||
if (cred != NULL) { | return (NULL); | ||||
object->cred = cred; | |||||
object->charge = size; | |||||
} | } | ||||
return (object); | return (object); | ||||
} | } | ||||
/* | /* | ||||
* SWAP_PAGER_ALLOC() - allocate a new OBJT_SWAP VM object and instantiate | * SWAP_PAGER_ALLOC() - allocate a new OBJT_SWAP VM object and instantiate | ||||
* its metadata structures. | * its metadata structures. | ||||
* | * | ||||
Show All 30 Lines | if (handle != NULL) { | ||||
sx_xunlock(&sw_alloc_sx); | sx_xunlock(&sw_alloc_sx); | ||||
} else { | } else { | ||||
object = swap_pager_alloc_init(OBJT_SWAP, handle, cred, | object = swap_pager_alloc_init(OBJT_SWAP, handle, cred, | ||||
size, offset); | size, offset); | ||||
} | } | ||||
return (object); | return (object); | ||||
} | } | ||||
static vm_object_t | |||||
swap_tmpfs_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, | |||||
vm_ooffset_t offset, struct ucred *cred) | |||||
{ | |||||
vm_object_t object; | |||||
MPASS(handle == NULL); | |||||
object = swap_pager_alloc_init(OBJT_SWAP_TMPFS, handle, cred, | |||||
size, offset); | |||||
return (object); | |||||
} | |||||
/* | /* | ||||
* SWAP_PAGER_DEALLOC() - remove swap metadata from object | * SWAP_PAGER_DEALLOC() - remove swap metadata from object | ||||
* | * | ||||
* The swap backing for the object is destroyed. The code is | * The swap backing for the object is destroyed. The code is | ||||
* designed such that we can reinstantiate it later, but this | * designed such that we can reinstantiate it later, but this | ||||
* routine is typically called only when the entire object is | * routine is typically called only when the entire object is | ||||
* about to be destroyed. | * about to be destroyed. | ||||
* | * | ||||
▲ Show 20 Lines • Show All 2,383 Lines • ▼ Show 20 Lines | swap_pager_release_writecount(vm_object_t object, vm_offset_t start, | ||||
vm_offset_t end) | vm_offset_t end) | ||||
{ | { | ||||
VM_OBJECT_WLOCK(object); | VM_OBJECT_WLOCK(object); | ||||
KASSERT((object->flags & OBJ_ANON) == 0, | KASSERT((object->flags & OBJ_ANON) == 0, | ||||
("Splittable object with writecount")); | ("Splittable object with writecount")); | ||||
object->un_pager.swp.writemappings -= (vm_ooffset_t)end - start; | object->un_pager.swp.writemappings -= (vm_ooffset_t)end - start; | ||||
VM_OBJECT_WUNLOCK(object); | VM_OBJECT_WUNLOCK(object); | ||||
} | |||||
static void | |||||
swap_tmpfs_pager_getvp(vm_object_t object, struct vnode **vpp, bool *vp_heldp) | |||||
{ | |||||
struct vnode *vp; | |||||
/* | |||||
* Tmpfs VREG node, which was reclaimed, has OBJT_SWAP_TMPFS | |||||
* type, but not OBJ_TMPFS flag. In this case there is no | |||||
* v_writecount to adjust. | |||||
*/ | |||||
if (vp_heldp != NULL) | |||||
VM_OBJECT_RLOCK(object); | |||||
else | |||||
VM_OBJECT_ASSERT_LOCKED(object); | |||||
if ((object->flags & OBJ_TMPFS) != 0) { | |||||
vp = object->un_pager.swp.swp_tmpfs; | |||||
if (vp != NULL) { | |||||
*vpp = vp; | |||||
if (vp_heldp != NULL) { | |||||
vhold(vp); | |||||
*vp_heldp = true; | |||||
} | |||||
} | |||||
} | |||||
if (vp_heldp != NULL) | |||||
VM_OBJECT_RUNLOCK(object); | |||||
} | } |