diff --git a/sys/fs/procfs/procfs_map.c b/sys/fs/procfs/procfs_map.c --- a/sys/fs/procfs/procfs_map.c +++ b/sys/fs/procfs/procfs_map.c @@ -51,6 +51,7 @@ #include #endif #include +#include #include #include @@ -85,7 +86,8 @@ char *fullpath, *freepath, *type; struct ucred *cred; vm_object_t lobj, nobj, obj, tobj; - int error, privateresident, ref_count, resident, shadow_count, flags; + int error, flags, kvme, privateresident, ref_count, resident; + int shadow_count; vm_offset_t e_start, e_end; vm_eflags_t e_eflags; vm_prot_t e_prot; @@ -155,29 +157,29 @@ freepath = NULL; fullpath = "-"; if (lobj) { - vp = NULL; - switch (lobj->type) { - default: - case OBJT_DEFAULT: - type = "default"; - break; - case OBJT_VNODE: - type = "vnode"; - vp = lobj->handle; + kvme = vm_object_kvme_type(lobj, &vp); + if (vp != NULL) vref(vp); + switch (kvme) { + default: + type = "unknown"; break; - case OBJT_SWAP_TMPFS: - type = "vnode"; - if ((lobj->flags & OBJ_TMPFS) != 0) { - vp = lobj->un_pager.swp.swp_tmpfs; - vref(vp); - } + case KVME_TYPE_PHYS: + type = "phys"; break; - case OBJT_SWAP: + case KVME_TYPE_DEFAULT: + case KVME_TYPE_SWAP: type = "swap"; break; - case OBJT_SG: - case OBJT_DEVICE: + case KVME_TYPE_DEAD: + type = "dead"; + break; + case KVME_TYPE_VNODE: + type = "vnode"; + break; + case KVME_TYPE_SG: + case KVME_TYPE_DEVICE: + case KVME_TYPE_MGTDEVICE: type = "device"; break; } diff --git a/sys/fs/tmpfs/tmpfs.h b/sys/fs/tmpfs/tmpfs.h --- a/sys/fs/tmpfs/tmpfs.h +++ b/sys/fs/tmpfs/tmpfs.h @@ -45,6 +45,8 @@ MALLOC_DECLARE(M_TMPFSNAME); #endif +#define OBJ_TMPFS OBJ_PAGERPRIV /* has tmpfs vnode allocated */ + /* * Internal representation of a tmpfs directory entry. */ @@ -514,9 +516,11 @@ size_t tmpfs_mem_avail(void); size_t tmpfs_pages_used(struct tmpfs_mount *tmp); -void tmpfs_subr_init(void); +int tmpfs_subr_init(void); void tmpfs_subr_uninit(void); +extern int tmpfs_pager_type; + /* * Macros/functions to convert from generic data structures to tmpfs * specific ones. diff --git a/sys/fs/tmpfs/tmpfs_subr.c b/sys/fs/tmpfs/tmpfs_subr.c --- a/sys/fs/tmpfs/tmpfs_subr.c +++ b/sys/fs/tmpfs/tmpfs_subr.c @@ -54,6 +54,7 @@ #include #include #include +#include #include #include @@ -79,6 +80,61 @@ static uma_zone_t tmpfs_node_pool; VFS_SMR_DECLARE; +int tmpfs_pager_type = -1; + +static vm_object_t +tmpfs_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, + vm_ooffset_t offset, struct ucred *cred) +{ + vm_object_t object; + + MPASS(handle == NULL); + MPASS(offset == 0); + object = vm_object_allocate_dyn(tmpfs_pager_type, size, + OBJ_COLORED | OBJ_SWAP); + if (!swap_pager_init_object(object, NULL, NULL, size, 0)) { + vm_object_deallocate(object); + object = NULL; + } + return (object); +} + +static void +tmpfs_pager_getvp(vm_object_t object, struct vnode **vpp, bool *vp_heldp) +{ + struct vnode *vp; + + /* + * Tmpfs VREG node, which was reclaimed, has tmpfs_pager_type + * type, but not OBJ_TMPFS flag. In this case there is no + * v_writecount to adjust. + */ + if (vp_heldp != NULL) + VM_OBJECT_RLOCK(object); + else + VM_OBJECT_ASSERT_LOCKED(object); + if ((object->flags & OBJ_TMPFS) != 0) { + vp = object->un_pager.swp.swp_tmpfs; + if (vp != NULL) { + *vpp = vp; + if (vp_heldp != NULL) { + vhold(vp); + *vp_heldp = true; + } + } + } + if (vp_heldp != NULL) + VM_OBJECT_RUNLOCK(object); +} + +struct pagerops tmpfs_pager_ops = { + .pgo_kvme_type = KVME_TYPE_VNODE, + .pgo_alloc = tmpfs_pager_alloc, + .pgo_set_writeable_dirty = vm_object_set_writeable_dirty_, + .pgo_mightbedirty = vm_object_mightbedirty_, + .pgo_getvp = tmpfs_pager_getvp, +}; + static int tmpfs_node_ctor(void *mem, int size, void *arg, int flags) { @@ -126,18 +182,26 @@ mtx_destroy(&node->tn_interlock); } -void +int tmpfs_subr_init(void) { + tmpfs_pager_type = vm_pager_alloc_dyn_type(&tmpfs_pager_ops, + OBJT_SWAP); + if (tmpfs_pager_type == -1) + return (EINVAL); tmpfs_node_pool = uma_zcreate("TMPFS node", sizeof(struct tmpfs_node), tmpfs_node_ctor, tmpfs_node_dtor, tmpfs_node_init, tmpfs_node_fini, UMA_ALIGN_PTR, 0); VFS_SMR_ZONE_SET(tmpfs_node_pool); + return (0); } void tmpfs_subr_uninit(void) { + if (tmpfs_pager_type != -1) + vm_pager_free_dyn_type(tmpfs_pager_type); + tmpfs_pager_type = -1; uma_zdestroy(tmpfs_node_pool); } @@ -364,7 +428,7 @@ case VREG: obj = nnode->tn_reg.tn_aobj = - vm_pager_allocate(OBJT_SWAP_TMPFS, NULL, 0, + vm_pager_allocate(tmpfs_pager_type, NULL, 0, VM_PROT_DEFAULT, 0, NULL /* XXXKIB - tmpfs needs swap reservation */); /* OBJ_TMPFS is set together with the setting of vp->v_object */ @@ -1588,7 +1652,7 @@ if (vp->v_type != VREG) return; obj = vp->v_object; - KASSERT(obj->type == OBJT_SWAP_TMPFS && + KASSERT(obj->type == tmpfs_pager_type && (obj->flags & (OBJ_SWAP | OBJ_TMPFS)) == (OBJ_SWAP | OBJ_TMPFS), ("non-tmpfs obj")); /* unlocked read */ diff --git a/sys/fs/tmpfs/tmpfs_vfsops.c b/sys/fs/tmpfs/tmpfs_vfsops.c --- a/sys/fs/tmpfs/tmpfs_vfsops.c +++ b/sys/fs/tmpfs/tmpfs_vfsops.c @@ -103,8 +103,8 @@ * Handle updates of time from writes to mmaped regions, if allowed. * Use MNT_VNODE_FOREACH_ALL instead of MNT_VNODE_FOREACH_LAZY, since * unmap of the tmpfs-backed vnode does not call vinactive(), due to - * vm object type is OBJT_SWAP. If lazy, only handle delayed update - * of mtime due to the writes to mapped files. + * vm object type is basically OBJT_SWAP. If lazy, only handle + * delayed update of mtime due to the writes to mapped files. */ static void tmpfs_update_mtime(struct mount *mp, bool lazy) @@ -120,7 +120,7 @@ continue; } obj = vp->v_object; - MPASS(obj->type == OBJT_SWAP_TMPFS); + MPASS(obj->type == tmpfs_pager_type); MPASS((obj->flags & OBJ_TMPFS) != 0); /* @@ -225,7 +225,7 @@ (entry->max_protection & VM_PROT_WRITE) == 0) continue; object = entry->object.vm_object; - if (object == NULL || object->type != OBJT_SWAP_TMPFS) + if (object == NULL || object->type != tmpfs_pager_type) continue; /* * No need to dig into shadow chain, mapping @@ -661,7 +661,11 @@ static int tmpfs_init(struct vfsconf *conf) { - tmpfs_subr_init(); + int res; + + res = tmpfs_subr_init(); + if (res != 0) + return (res); memcpy(&tmpfs_fnops, &vnops, sizeof(struct fileops)); tmpfs_fnops.fo_close = tmpfs_fo_close; return (0); diff --git a/sys/fs/tmpfs/tmpfs_vnops.c b/sys/fs/tmpfs/tmpfs_vnops.c --- a/sys/fs/tmpfs/tmpfs_vnops.c +++ b/sys/fs/tmpfs/tmpfs_vnops.c @@ -623,7 +623,7 @@ if (object == NULL) goto out_smr; - MPASS(object->type == OBJT_SWAP_TMPFS); + MPASS(object->type == tmpfs_pager_type); MPASS((object->flags & (OBJ_ANON | OBJ_DEAD | OBJ_SWAP)) == OBJ_SWAP); if (!VN_IS_DOOMED(vp)) { diff --git a/sys/vm/default_pager.c b/sys/vm/default_pager.c --- a/sys/vm/default_pager.c +++ b/sys/vm/default_pager.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include @@ -71,6 +72,7 @@ * object is converted to swap pager type. */ const struct pagerops defaultpagerops = { + .pgo_kvme_type = KVME_TYPE_DEFAULT, .pgo_alloc = default_pager_alloc, .pgo_dealloc = default_pager_dealloc, .pgo_getpages = default_pager_getpages, diff --git a/sys/vm/device_pager.c b/sys/vm/device_pager.c --- a/sys/vm/device_pager.c +++ b/sys/vm/device_pager.c @@ -48,6 +48,7 @@ #include #include #include +#include #include #include @@ -75,6 +76,7 @@ static struct mtx dev_pager_mtx; const struct pagerops devicepagerops = { + .pgo_kvme_type = KVME_TYPE_DEVICE, .pgo_init = dev_pager_init, .pgo_alloc = dev_pager_alloc, .pgo_dealloc = dev_pager_dealloc, @@ -84,6 +86,7 @@ }; const struct pagerops mgtdevicepagerops = { + .pgo_kvme_type = KVME_TYPE_MGTDEVICE, .pgo_alloc = dev_pager_alloc, .pgo_dealloc = dev_pager_dealloc, .pgo_getpages = dev_pager_getpages, diff --git a/sys/vm/phys_pager.c b/sys/vm/phys_pager.c --- a/sys/vm/phys_pager.c +++ b/sys/vm/phys_pager.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -299,6 +300,7 @@ } const struct pagerops physpagerops = { + .pgo_kvme_type = KVME_TYPE_PHYS, .pgo_init = phys_pager_init, .pgo_alloc = phys_pager_alloc, .pgo_dealloc = phys_pager_dealloc, diff --git a/sys/vm/sg_pager.c b/sys/vm/sg_pager.c --- a/sys/vm/sg_pager.c +++ b/sys/vm/sg_pager.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include @@ -60,6 +61,7 @@ int *); const struct pagerops sgpagerops = { + .pgo_kvme_type = KVME_TYPE_SG, .pgo_alloc = sg_pager_alloc, .pgo_dealloc = sg_pager_dealloc, .pgo_getpages = sg_pager_getpages, diff --git a/sys/vm/swap_pager.h b/sys/vm/swap_pager.h --- a/sys/vm/swap_pager.h +++ b/sys/vm/swap_pager.h @@ -82,6 +82,7 @@ void swap_pager_status(int *total, int *used); u_long swap_pager_swapped_pages(vm_object_t object); void swapoff_all(void); - +bool swap_pager_init_object(vm_object_t object, void *handle, + struct ucred *cred, vm_ooffset_t size, vm_ooffset_t offset); #endif /* _KERNEL */ #endif /* _VM_SWAP_PAGER_H_ */ diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c --- a/sys/vm/swap_pager.c +++ b/sys/vm/swap_pager.c @@ -99,6 +99,7 @@ #include #include #include +#include #include #include @@ -418,9 +419,6 @@ static vm_object_t swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t offset, struct ucred *); -static vm_object_t - swap_tmpfs_pager_alloc(void *handle, vm_ooffset_t size, - vm_prot_t prot, vm_ooffset_t offset, struct ucred *); static void swap_pager_dealloc(vm_object_t object); static int swap_pager_getpages(vm_object_t, vm_page_t *, int, int *, int *); @@ -436,12 +434,11 @@ vm_offset_t start, vm_offset_t end); static void swap_pager_release_writecount(vm_object_t object, vm_offset_t start, vm_offset_t end); -static void swap_tmpfs_pager_getvp(vm_object_t object, struct vnode **vpp, - bool *vp_heldp); static void swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size); const struct pagerops swappagerops = { + .pgo_kvme_type = KVME_TYPE_SWAP, .pgo_init = swap_pager_init, /* early system initialization of pager */ .pgo_alloc = swap_pager_alloc, /* allocate an OBJT_SWAP object */ .pgo_dealloc = swap_pager_dealloc, /* deallocate an OBJT_SWAP object */ @@ -455,22 +452,6 @@ .pgo_freespace = swap_pager_freespace, }; -const struct pagerops swaptmpfspagerops = { - .pgo_alloc = swap_tmpfs_pager_alloc, - .pgo_dealloc = swap_pager_dealloc, - .pgo_getpages = swap_pager_getpages, - .pgo_getpages_async = swap_pager_getpages_async, - .pgo_putpages = swap_pager_putpages, - .pgo_haspage = swap_pager_haspage, - .pgo_pageunswapped = swap_pager_unswapped, - .pgo_update_writecount = swap_pager_update_writecount, - .pgo_release_writecount = swap_pager_release_writecount, - .pgo_set_writeable_dirty = vm_object_set_writeable_dirty_, - .pgo_mightbedirty = vm_object_mightbedirty_, - .pgo_getvp = swap_tmpfs_pager_getvp, - .pgo_freespace = swap_pager_freespace, -}; - /* * swap_*() routines are externally accessible. swp_*() routines are * internal. @@ -678,18 +659,31 @@ "reduce kern.maxswzone.\n"); } -static vm_object_t -swap_pager_alloc_init(objtype_t otype, void *handle, struct ucred *cred, +bool +swap_pager_init_object(vm_object_t object, void *handle, struct ucred *cred, vm_ooffset_t size, vm_ooffset_t offset) { - vm_object_t object; - if (cred != NULL) { if (!swap_reserve_by_cred(size, cred)) - return (NULL); + return (false); crhold(cred); } + object->un_pager.swp.writemappings = 0; + object->handle = handle; + if (cred != NULL) { + object->cred = cred; + object->charge = size; + } + return (true); +} + +static vm_object_t +swap_pager_alloc_init(objtype_t otype, void *handle, struct ucred *cred, + vm_ooffset_t size, vm_ooffset_t offset) +{ + vm_object_t object; + /* * The un_pager.swp.swp_blks trie is initialized by * vm_object_allocate() to ensure the correct order of @@ -698,11 +692,9 @@ object = vm_object_allocate(otype, OFF_TO_IDX(offset + PAGE_MASK + size)); - object->un_pager.swp.writemappings = 0; - object->handle = handle; - if (cred != NULL) { - object->cred = cred; - object->charge = size; + if (!swap_pager_init_object(object, handle, cred, size, offset)) { + vm_object_deallocate(object); + return (NULL); } return (object); } @@ -749,18 +741,6 @@ return (object); } -static vm_object_t -swap_tmpfs_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, - vm_ooffset_t offset, struct ucred *cred) -{ - vm_object_t object; - - MPASS(handle == NULL); - object = swap_pager_alloc_init(OBJT_SWAP_TMPFS, handle, cred, - size, offset); - return (object); -} - /* * SWAP_PAGER_DEALLOC() - remove swap metadata from object * @@ -3161,31 +3141,3 @@ object->un_pager.swp.writemappings -= (vm_ooffset_t)end - start; VM_OBJECT_WUNLOCK(object); } - -static void -swap_tmpfs_pager_getvp(vm_object_t object, struct vnode **vpp, bool *vp_heldp) -{ - struct vnode *vp; - - /* - * Tmpfs VREG node, which was reclaimed, has OBJT_SWAP_TMPFS - * type, but not OBJ_TMPFS flag. In this case there is no - * v_writecount to adjust. - */ - if (vp_heldp != NULL) - VM_OBJECT_RLOCK(object); - else - VM_OBJECT_ASSERT_LOCKED(object); - if ((object->flags & OBJ_TMPFS) != 0) { - vp = object->un_pager.swp.swp_tmpfs; - if (vp != NULL) { - *vpp = vp; - if (vp_heldp != NULL) { - vhold(vp); - *vp_heldp = true; - } - } - } - if (vp_heldp != NULL) - VM_OBJECT_RUNLOCK(object); -} diff --git a/sys/vm/vm.h b/sys/vm/vm.h --- a/sys/vm/vm.h +++ b/sys/vm/vm.h @@ -97,7 +97,7 @@ OBJT_DEAD, OBJT_SG, OBJT_MGTDEVICE, - OBJT_SWAP_TMPFS, + OBJT_FIRST_DYN, }; typedef u_char objtype_t; diff --git a/sys/vm/vm_object.h b/sys/vm/vm_object.h --- a/sys/vm/vm_object.h +++ b/sys/vm/vm_object.h @@ -207,7 +207,7 @@ #define OBJ_COLORED 0x1000 /* pg_color is defined */ #define OBJ_ONEMAPPING 0x2000 /* One USE (a single, non-forked) mapping flag */ #define OBJ_SHADOWLIST 0x4000 /* Object is on the shadow list. */ -#define OBJ_TMPFS 0x8000 /* has tmpfs vnode allocated */ +#define OBJ_PAGERPRIV 0x8000 /* Pager private */ /* * Helpers to perform conversion between vm_object page indexes and offsets. @@ -356,6 +356,7 @@ vm_object_t vm_object_allocate (objtype_t, vm_pindex_t); vm_object_t vm_object_allocate_anon(vm_pindex_t, vm_object_t, struct ucred *, vm_size_t); +vm_object_t vm_object_allocate_dyn(objtype_t, vm_pindex_t, u_short); boolean_t vm_object_coalesce(vm_object_t, vm_ooffset_t, vm_size_t, vm_size_t, boolean_t); void vm_object_collapse (vm_object_t); diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c --- a/sys/vm/vm_object.c +++ b/sys/vm/vm_object.c @@ -330,24 +330,12 @@ { VM_OBJECT_ASSERT_WLOCKED(object); - switch (object->type) { - case OBJT_DEFAULT: - case OBJT_DEVICE: - case OBJT_MGTDEVICE: - case OBJT_PHYS: - case OBJT_SG: - case OBJT_SWAP: - case OBJT_SWAP_TMPFS: - case OBJT_VNODE: - if (!TAILQ_EMPTY(&object->memq)) - return (KERN_FAILURE); - break; - case OBJT_DEAD: + + if (object->type == OBJT_DEAD) return (KERN_INVALID_ARGUMENT); - default: - panic("vm_object_set_memattr: object %p is of undefined type", - object); - } + if (!TAILQ_EMPTY(&object->memq)) + return (KERN_FAILURE); + object->memattr = memattr; return (KERN_SUCCESS); } @@ -425,7 +413,6 @@ flags = OBJ_COLORED; break; case OBJT_SWAP: - case OBJT_SWAP_TMPFS: flags = OBJ_COLORED | OBJ_SWAP; break; case OBJT_DEVICE: @@ -442,7 +429,8 @@ flags = 0; break; default: - panic("vm_object_allocate: type %d is undefined", type); + panic("vm_object_allocate: type %d is undefined or dynamic", + type); } object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK); _vm_object_allocate(type, size, flags, object, NULL); @@ -450,6 +438,18 @@ return (object); } +vm_object_t +vm_object_allocate_dyn(objtype_t dyntype, vm_pindex_t size, u_short flags) +{ + vm_object_t object; + + MPASS(dyntype >= OBJT_FIRST_DYN /* && dyntype < nitems(pagertab) */); + object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK); + _vm_object_allocate(dyntype, size, flags, object, NULL); + + return (object); +} + /* * vm_object_allocate_anon: * @@ -681,7 +681,8 @@ umtx_shm_object_terminated(object); temp = object->backing_object; if (temp != NULL) { - KASSERT(object->type != OBJT_SWAP_TMPFS, + KASSERT(object->type == OBJT_DEFAULT || + object->type == OBJT_SWAP, ("shadowed tmpfs v_object 2 %p", object)); vm_object_backing_remove(object); } @@ -962,7 +963,7 @@ #endif KASSERT(object->cred == NULL || object->type == OBJT_DEFAULT || - object->type == OBJT_SWAP || object->type == OBJT_SWAP_TMPFS, + (object->flags & OBJ_SWAP) != 0, ("%s: non-swap obj %p has cred", __func__, object)); /* @@ -2469,41 +2470,6 @@ (void)blockcount_sleep(&obj->busy, NULL, wmesg, PVM); } -/* - * Return the kvme type of the given object. - * If vpp is not NULL, set it to the object's vm_object_vnode() or NULL. - */ -int -vm_object_kvme_type(vm_object_t object, struct vnode **vpp) -{ - - VM_OBJECT_ASSERT_LOCKED(object); - if (vpp != NULL) - *vpp = vm_object_vnode(object); - switch (object->type) { - case OBJT_DEFAULT: - return (KVME_TYPE_DEFAULT); - case OBJT_VNODE: - return (KVME_TYPE_VNODE); - case OBJT_SWAP: - return (KVME_TYPE_SWAP); - case OBJT_SWAP_TMPFS: - return (KVME_TYPE_VNODE); - case OBJT_DEVICE: - return (KVME_TYPE_DEVICE); - case OBJT_PHYS: - return (KVME_TYPE_PHYS); - case OBJT_DEAD: - return (KVME_TYPE_DEAD); - case OBJT_SG: - return (KVME_TYPE_SG); - case OBJT_MGTDEVICE: - return (KVME_TYPE_MGTDEVICE); - default: - return (KVME_TYPE_UNKNOWN); - } -} - static int sysctl_vm_object_list(SYSCTL_HANDLER_ARGS) { diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -1887,15 +1887,9 @@ if ((entry->eflags & MAP_ENTRY_NEEDS_COPY) != 0 && obj->ref_count != 1) continue; - switch (obj->type) { - case OBJT_DEFAULT: - case OBJT_SWAP: - case OBJT_SWAP_TMPFS: - case OBJT_PHYS: - case OBJT_VNODE: + if (obj->type == OBJT_DEFAULT || obj->type == OBJT_PHYS || + obj->type == OBJT_VNODE || (obj->flags & OBJ_SWAP) != 0) res += obj->resident_page_count; - break; - } } return (res); } diff --git a/sys/vm/vm_pager.h b/sys/vm/vm_pager.h --- a/sys/vm/vm_pager.h +++ b/sys/vm/vm_pager.h @@ -71,6 +71,7 @@ vm_size_t size); struct pagerops { + int pgo_kvme_type; pgo_init_t *pgo_init; /* Initialize pager. */ pgo_alloc_t *pgo_alloc; /* Allocate pager. */ pgo_dealloc_t *pgo_dealloc; /* Disassociate. */ @@ -248,6 +249,9 @@ method(object, start, size); } +int vm_pager_alloc_dyn_type(struct pagerops *ops, int base_type); +void vm_pager_free_dyn_type(objtype_t type); + struct cdev_pager_ops { int (*cdev_pg_fault)(vm_object_t vm_obj, vm_ooffset_t offset, int prot, vm_page_t *mres); diff --git a/sys/vm/vm_pager.c b/sys/vm/vm_pager.c --- a/sys/vm/vm_pager.c +++ b/sys/vm/vm_pager.c @@ -79,6 +79,7 @@ #include #include #include +#include #include #include @@ -155,6 +156,7 @@ } static const struct pagerops deadpagerops = { + .pgo_kvme_type = KVME_TYPE_DEAD, .pgo_alloc = dead_pager_alloc, .pgo_dealloc = dead_pager_dealloc, .pgo_getpages = dead_pager_getpages, @@ -163,7 +165,7 @@ .pgo_getvp = dead_pager_getvp, }; -const struct pagerops *pagertab[] __read_mostly = { +const struct pagerops *pagertab[16] __read_mostly = { [OBJT_DEFAULT] = &defaultpagerops, [OBJT_SWAP] = &swappagerops, [OBJT_VNODE] = &vnodepagerops, @@ -172,20 +174,25 @@ [OBJT_DEAD] = &deadpagerops, [OBJT_SG] = &sgpagerops, [OBJT_MGTDEVICE] = &mgtdevicepagerops, - [OBJT_SWAP_TMPFS] = &swaptmpfspagerops, }; +static struct mtx pagertab_lock; void vm_pager_init(void) { const struct pagerops **pgops; + int i; + + mtx_init(&pagertab_lock, "dynpag", NULL, MTX_DEF); /* * Initialize known pagers */ - for (pgops = pagertab; pgops < &pagertab[nitems(pagertab)]; pgops++) + for (i = 0; i < OBJT_FIRST_DYN; i++) { + pgops = &pagertab[i]; if ((*pgops)->pgo_init != NULL) (*(*pgops)->pgo_init)(); + } } static int nswbuf_max; @@ -243,15 +250,9 @@ vm_pager_allocate(objtype_t type, void *handle, vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t off, struct ucred *cred) { - vm_object_t ret; - const struct pagerops *ops; + MPASS(type < nitems(pagertab)); - ops = pagertab[type]; - if (ops) - ret = (*ops->pgo_alloc)(handle, size, prot, off, cred); - else - ret = NULL; - return (ret); + return ((*pagertab[type]->pgo_alloc)(handle, size, prot, off, cred)); } /* @@ -262,6 +263,7 @@ { VM_OBJECT_ASSERT_WLOCKED(object); + MPASS(object->type < nitems(pagertab)); (*pagertab[object->type]->pgo_dealloc) (object); } @@ -313,6 +315,7 @@ #endif int r; + MPASS(object->type < nitems(pagertab)); vm_pager_assert_in(object, m, count); r = (*pagertab[object->type]->pgo_getpages)(object, m, count, rbehind, @@ -346,6 +349,7 @@ int *rbehind, int *rahead, pgo_getpages_iodone_t iodone, void *arg) { + MPASS(object->type < nitems(pagertab)); vm_pager_assert_in(object, m, count); return ((*pagertab[object->type]->pgo_getpages_async)(object, m, @@ -383,6 +387,60 @@ return (object); } +int +vm_pager_alloc_dyn_type(struct pagerops *ops, int base_type) +{ + int res; + + mtx_lock(&pagertab_lock); + MPASS(base_type == -1 || + (base_type >= OBJT_DEFAULT && base_type < nitems(pagertab))); + for (res = OBJT_FIRST_DYN; res < nitems(pagertab); res++) { + if (pagertab[res] == NULL) + break; + } + if (res == nitems(pagertab)) { + mtx_unlock(&pagertab_lock); + return (-1); + } + if (base_type != -1) { + MPASS(pagertab[base_type] != NULL); +#define FIX(n) \ + if (ops->pgo_##n == NULL) \ + ops->pgo_##n = pagertab[base_type]->pgo_##n + FIX(init); + FIX(alloc); + FIX(dealloc); + FIX(getpages); + FIX(getpages_async); + FIX(putpages); + FIX(haspage); + FIX(populate); + FIX(pageunswapped); + FIX(update_writecount); + FIX(release_writecount); + FIX(set_writeable_dirty); + FIX(mightbedirty); + FIX(getvp); + FIX(freespace); +#undef FIX + } + pagertab[res] = ops; /* XXXKIB should be rel, but acq is too much */ + mtx_unlock(&pagertab_lock); + return (res); +} + +void +vm_pager_free_dyn_type(objtype_t type) +{ + MPASS(type >= OBJT_FIRST_DYN && type < nitems(pagertab)); + + mtx_lock(&pagertab_lock); + MPASS(pagertab[type] != NULL); + pagertab[type] = NULL; + mtx_unlock(&pagertab_lock); +} + static int pbuf_ctor(void *mem, int size, void *arg, int flags) { @@ -515,6 +573,8 @@ { pgo_set_writeable_dirty_t *method; + MPASS(object->type < nitems(pagertab)); + method = pagertab[object->type]->pgo_set_writeable_dirty; if (method != NULL) method(object); @@ -525,8 +585,25 @@ { pgo_mightbedirty_t *method; + MPASS(object->type < nitems(pagertab)); + method = pagertab[object->type]->pgo_mightbedirty; if (method == NULL) return (false); return (method(object)); } + +/* + * Return the kvme type of the given object. + * If vpp is not NULL, set it to the object's vm_object_vnode() or NULL. + */ +int +vm_object_kvme_type(vm_object_t object, struct vnode **vpp) +{ + VM_OBJECT_ASSERT_LOCKED(object); + MPASS(object->type < nitems(pagertab)); + + if (vpp != NULL) + *vpp = vm_object_vnode(object); + return (pagertab[object->type]->pgo_kvme_type); +} diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c --- a/sys/vm/vnode_pager.c +++ b/sys/vm/vnode_pager.c @@ -74,6 +74,7 @@ #include #include #include +#include #include @@ -108,6 +109,7 @@ static void vnode_pager_getvp(vm_object_t, struct vnode **, bool *); const struct pagerops vnodepagerops = { + .pgo_kvme_type = KVME_TYPE_VNODE, .pgo_alloc = vnode_pager_alloc, .pgo_dealloc = vnode_pager_dealloc, .pgo_getpages = vnode_pager_getpages,