Index: sys/kern/uipc_shm.c =================================================================== --- sys/kern/uipc_shm.c +++ sys/kern/uipc_shm.c @@ -896,8 +896,11 @@ struct shmfd *shmfd; vm_prot_t maxprot; int error; + bool writecnt; shmfd = fp->f_data; + writecnt = false; + maxprot = VM_PROT_NONE; /* FREAD should always be set. */ @@ -906,10 +909,11 @@ if ((fp->f_flag & FWRITE) != 0) maxprot |= VM_PROT_WRITE; + if ((flags & MAP_SHARED) != 0 && (prot & VM_PROT_WRITE) != 0) + writecnt = true; + /* Don't permit shared writable mappings on read-only descriptors. */ - if ((flags & MAP_SHARED) != 0 && - (maxprot & VM_PROT_WRITE) == 0 && - (prot & VM_PROT_WRITE) != 0) + if (writecnt && (maxprot & VM_PROT_WRITE) == 0) return (EACCES); maxprot &= cap_maxprot; @@ -932,10 +936,16 @@ mtx_unlock(&shm_timestamp_lock); vm_object_reference(shmfd->shm_object); + if (writecnt) + vm_pager_update_writecount(shmfd->shm_object, 0, objsize); error = vm_mmap_object(map, addr, objsize, prot, maxprot, flags, - shmfd->shm_object, foff, FALSE, td); - if (error != 0) + shmfd->shm_object, foff, writecnt, td); + if (error != 0) { + if (writecnt) + vm_pager_release_writecount(shmfd->shm_object, 0, + objsize); vm_object_deallocate(shmfd->shm_object); + } return (error); } Index: sys/kern/vfs_vnops.c =================================================================== --- sys/kern/vfs_vnops.c +++ sys/kern/vfs_vnops.c @@ -87,7 +87,7 @@ #include #include #include -#include +#include #ifdef HWPMC_HOOKS #include @@ -2502,7 +2502,7 @@ * writecount, then undo that now. */ if (writecounted) - vnode_pager_release_writecount(object, 0, size); + vm_pager_release_writecount(object, 0, size); vm_object_deallocate(object); } #ifdef HWPMC_HOOKS Index: sys/vm/vm_map.h =================================================================== --- sys/vm/vm_map.h +++ sys/vm/vm_map.h @@ -144,8 +144,7 @@ #define MAP_ENTRY_GROWS_UP 0x00002000 /* bottom-up stacks */ #define MAP_ENTRY_WIRE_SKIPPED 0x00004000 -#define MAP_ENTRY_VN_WRITECNT 0x00008000 /* writeable vnode - mapping */ +#define MAP_ENTRY_WRITECNT 0x00008000 /* writeable mapping */ #define MAP_ENTRY_GUARD 0x00010000 #define MAP_ENTRY_STACK_GAP_DN 0x00020000 #define MAP_ENTRY_STACK_GAP_UP 0x00040000 @@ -348,7 +347,7 @@ #define MAP_CREATE_GUARD 0x00000080 #define MAP_DISABLE_COREDUMP 0x00000100 #define MAP_PREFAULT_MADVISE 0x00000200 /* from (user) madvise request */ -#define MAP_VN_WRITECOUNT 0x00000400 +#define MAP_WRITECOUNT 0x00000400 #define MAP_REMAP 0x00000800 #define MAP_STACK_GROWS_DOWN 0x00001000 #define MAP_STACK_GROWS_UP 0x00002000 Index: sys/vm/vm_map.c =================================================================== --- sys/vm/vm_map.c +++ sys/vm/vm_map.c @@ -575,10 +575,10 @@ td->td_map_def_user = NULL; while (entry != NULL) { next = entry->next; - MPASS((entry->eflags & (MAP_ENTRY_VN_WRITECNT | - MAP_ENTRY_VN_EXEC)) != (MAP_ENTRY_VN_WRITECNT | + MPASS((entry->eflags & (MAP_ENTRY_WRITECNT | + MAP_ENTRY_VN_EXEC)) != (MAP_ENTRY_WRITECNT | MAP_ENTRY_VN_EXEC)); - if ((entry->eflags & MAP_ENTRY_VN_WRITECNT) != 0) { + if ((entry->eflags & MAP_ENTRY_WRITECNT) != 0) { /* * Decrement the object's writemappings and * possibly the vnode's v_writecount. @@ -587,7 +587,7 @@ ("Submap with writecount")); object = entry->object.vm_object; KASSERT(object != NULL, ("No object for writecount")); - vnode_pager_release_writecount(object, entry->start, + vm_pager_release_writecount(object, entry->start, entry->end); } vm_map_entry_set_vnode_text(entry, false); @@ -1481,8 +1481,8 @@ protoeflags |= MAP_ENTRY_GROWS_DOWN; if (cow & MAP_STACK_GROWS_UP) protoeflags |= MAP_ENTRY_GROWS_UP; - if (cow & MAP_VN_WRITECOUNT) - protoeflags |= MAP_ENTRY_VN_WRITECNT; + if (cow & MAP_WRITECOUNT) + protoeflags |= MAP_ENTRY_WRITECNT; if (cow & MAP_VN_EXEC) protoeflags |= MAP_ENTRY_VN_EXEC; if ((cow & MAP_CREATE_GUARD) != 0) @@ -2073,7 +2073,7 @@ * kept without causing a lock-order reversal with the vnode lock. * * Since we count the number of virtual page mappings in - * object->un_pager.vnp.writemappings, the writemappings value + * object->writemappings, the writemappings value * should not be adjusted when the entry is disposed of. */ if (entry->object.vm_object != NULL) @@ -2210,8 +2210,8 @@ vm_object_reference(new_entry->object.vm_object); vm_map_entry_set_vnode_text(new_entry, true); /* - * The object->un_pager.vnp.writemappings for the - * object of MAP_ENTRY_VN_WRITECNT type entry shall be + * The object->writemappings for the + * object of MAP_ENTRY_WRITECNT type entry shall be * kept as is here. The virtual pages are * re-distributed among the clipped entries, so the sum is * left the same. @@ -3783,20 +3783,20 @@ dst_entry->eflags |= MAP_ENTRY_COW | MAP_ENTRY_NEEDS_COPY; dst_entry->offset = src_entry->offset; - if (src_entry->eflags & MAP_ENTRY_VN_WRITECNT) { + if (src_entry->eflags & MAP_ENTRY_WRITECNT) { /* - * MAP_ENTRY_VN_WRITECNT cannot + * MAP_ENTRY_WRITECNT cannot * indicate write reference from * src_entry, since the entry is * marked as needs copy. Allocate a * fake entry that is used to - * decrement object->un_pager.vnp.writecount + * decrement object->writecount * at the appropriate time. Attach * fake_entry to the deferred list. */ fake_entry = vm_map_entry_create(dst_map); - fake_entry->eflags = MAP_ENTRY_VN_WRITECNT; - src_entry->eflags &= ~MAP_ENTRY_VN_WRITECNT; + fake_entry->eflags = MAP_ENTRY_WRITECNT; + src_entry->eflags &= ~MAP_ENTRY_WRITECNT; vm_object_reference(src_object); fake_entry->object.vm_object = src_object; fake_entry->start = src_entry->start; @@ -3971,12 +3971,12 @@ * not relock it later for the assertion * correctness. */ - if (old_entry->eflags & MAP_ENTRY_VN_WRITECNT && + if (old_entry->eflags & MAP_ENTRY_WRITECNT && object->type == OBJT_VNODE) { KASSERT(((struct vnode *)object->handle)-> v_writecount > 0, ("vmspace_fork: v_writecount %p", object)); - KASSERT(object->un_pager.vnp.writemappings > 0, + KASSERT(object->writemappings > 0, ("vmspace_fork: vnp.writecount %p", object)); } @@ -3991,8 +3991,8 @@ MAP_ENTRY_IN_TRANSITION); new_entry->wiring_thread = NULL; new_entry->wired_count = 0; - if (new_entry->eflags & MAP_ENTRY_VN_WRITECNT) { - vnode_pager_update_writecount(object, + if (new_entry->eflags & MAP_ENTRY_WRITECNT) { + vm_pager_update_writecount(object, new_entry->start, new_entry->end); } vm_map_entry_set_vnode_text(new_entry, true); @@ -4023,7 +4023,7 @@ * Copied entry is COW over the old object. */ new_entry->eflags &= ~(MAP_ENTRY_USER_WIRED | - MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_VN_WRITECNT); + MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_WRITECNT); new_entry->wiring_thread = NULL; new_entry->wired_count = 0; new_entry->object.vm_object = NULL; @@ -4047,7 +4047,7 @@ new_entry->end = old_entry->end; new_entry->eflags = old_entry->eflags & ~(MAP_ENTRY_USER_WIRED | MAP_ENTRY_IN_TRANSITION | - MAP_ENTRY_VN_WRITECNT | MAP_ENTRY_VN_EXEC); + MAP_ENTRY_WRITECNT | MAP_ENTRY_VN_EXEC); new_entry->protection = old_entry->protection; new_entry->max_protection = old_entry->max_protection; new_entry->inheritance = VM_INHERIT_ZERO; Index: sys/vm/vm_mmap.c =================================================================== --- sys/vm/vm_mmap.c +++ sys/vm/vm_mmap.c @@ -1289,7 +1289,7 @@ } if (writex) { *writecounted = TRUE; - vnode_pager_update_writecount(obj, 0, objsize); + vm_pager_update_writecount(obj, 0, objsize); } } else { error = EINVAL; @@ -1345,7 +1345,7 @@ done: if (error != 0 && *writecounted) { *writecounted = FALSE; - vnode_pager_update_writecount(obj, objsize, 0); + vm_pager_update_writecount(obj, objsize, 0); } vput(vp); return (error); @@ -1480,7 +1480,7 @@ * writecount, then undo that now. */ if (writecounted) - vnode_pager_release_writecount(object, 0, size); + vm_pager_release_writecount(object, 0, size); vm_object_deallocate(object); } return (error); @@ -1569,7 +1569,7 @@ if (flags & MAP_SHARED) docow |= MAP_INHERIT_SHARE; if (writecounted) - docow |= MAP_VN_WRITECOUNT; + docow |= MAP_WRITECOUNT; if (flags & MAP_STACK) { if (object != NULL) return (EINVAL); Index: sys/vm/vm_object.h =================================================================== --- sys/vm/vm_object.h +++ sys/vm/vm_object.h @@ -126,7 +126,6 @@ */ struct { off_t vnp_size; - vm_ooffset_t writemappings; } vnp; /* @@ -169,6 +168,7 @@ struct pctrie swp_blks; } swp; } un_pager; + vm_ooffset_t writemappings; struct ucred *cred; vm_ooffset_t charge; void *umtx_data; Index: sys/vm/vm_object.c =================================================================== --- sys/vm/vm_object.c +++ sys/vm/vm_object.c @@ -281,6 +281,7 @@ object->memattr = VM_MEMATTR_DEFAULT; object->cred = NULL; object->charge = 0; + object->writemappings = 0; object->handle = NULL; object->backing_object = NULL; object->backing_object_offset = (vm_ooffset_t) 0; Index: sys/vm/vm_pager.h =================================================================== --- sys/vm/vm_pager.h +++ sys/vm/vm_pager.h @@ -61,6 +61,7 @@ typedef int pgo_populate_t(vm_object_t, vm_pindex_t, int, vm_prot_t, vm_pindex_t *, vm_pindex_t *); typedef void pgo_pageunswapped_t(vm_page_t); +typedef void pgo_writecount_t(vm_object_t, vm_offset_t, vm_offset_t); struct pagerops { pgo_init_t *pgo_init; /* Initialize pager. */ @@ -72,6 +73,9 @@ pgo_haspage_t *pgo_haspage; /* Query page. */ pgo_populate_t *pgo_populate; /* Bulk spec pagein. */ pgo_pageunswapped_t *pgo_pageunswapped; + /* Operations for specialized writecount handling */ + pgo_writecount_t *pgo_update_writecount; + pgo_writecount_t *pgo_release_writecount; }; extern struct pagerops defaultpagerops; @@ -192,6 +196,36 @@ (*pagertab[m->object->type]->pgo_pageunswapped)(m); } +static __inline void +vm_pager_update_writecount(vm_object_t object, vm_offset_t start, + vm_offset_t end) +{ + + if (pagertab[object->type]->pgo_update_writecount) { + pagertab[object->type]->pgo_update_writecount(object, start, + end); + } else { + VM_OBJECT_WLOCK(object); + object->writemappings += (vm_ooffset_t)end - start; + VM_OBJECT_WUNLOCK(object); + } +} + +static __inline void +vm_pager_release_writecount(vm_object_t object, vm_offset_t start, + vm_offset_t end) +{ + + if (pagertab[object->type]->pgo_release_writecount) { + pagertab[object->type]->pgo_release_writecount(object, start, + end); + } else { + VM_OBJECT_WLOCK(object); + object->writemappings -= (vm_ooffset_t)end - start; + VM_OBJECT_WUNLOCK(object); + } +} + struct cdev_pager_ops { int (*cdev_pg_fault)(vm_object_t vm_obj, vm_ooffset_t offset, int prot, vm_page_t *mres); Index: sys/vm/vnode_pager.h =================================================================== --- sys/vm/vnode_pager.h +++ sys/vm/vnode_pager.h @@ -50,12 +50,8 @@ int vnode_pager_local_getpages(struct vop_getpages_args *ap); int vnode_pager_local_getpages_async(struct vop_getpages_async_args *ap); int vnode_pager_putpages_ioflags(int pager_flags); -void vnode_pager_release_writecount(vm_object_t object, vm_offset_t start, - vm_offset_t end); void vnode_pager_undirty_pages(vm_page_t *ma, int *rtvals, int written, off_t eof, int lpos); -void vnode_pager_update_writecount(vm_object_t object, vm_offset_t start, - vm_offset_t end); #endif /* _KERNEL */ #endif /* _VNODE_PAGER_ */ Index: sys/vm/vnode_pager.c =================================================================== --- sys/vm/vnode_pager.c +++ sys/vm/vnode_pager.c @@ -100,6 +100,10 @@ vm_ooffset_t, struct ucred *cred); static int vnode_pager_generic_getpages_done(struct buf *); static void vnode_pager_generic_getpages_done_async(struct buf *); +static void vnode_pager_update_writecount(vm_object_t, vm_offset_t, + vm_offset_t); +static void vnode_pager_release_writecount(vm_object_t, vm_offset_t, + vm_offset_t); struct pagerops vnodepagerops = { .pgo_alloc = vnode_pager_alloc, @@ -108,6 +112,8 @@ .pgo_getpages_async = vnode_pager_getpages_async, .pgo_putpages = vnode_pager_putpages, .pgo_haspage = vnode_pager_haspage, + .pgo_update_writecount = vnode_pager_update_writecount, + .pgo_release_writecount = vnode_pager_release_writecount, }; static struct domainset *vnode_domainset = NULL; @@ -283,7 +289,6 @@ object = vm_object_allocate(OBJT_VNODE, OFF_TO_IDX(round_page(size))); object->un_pager.vnp.vnp_size = size; - object->un_pager.vnp.writemappings = 0; object->domain.dr_policy = vnode_domainset; object->handle = handle; @@ -339,8 +344,8 @@ wakeup(object); } ASSERT_VOP_ELOCKED(vp, "vnode_pager_dealloc"); - if (object->un_pager.vnp.writemappings > 0) { - object->un_pager.vnp.writemappings = 0; + if (object->writemappings > 0) { + object->writemappings = 0; VOP_ADD_WRITECOUNT_CHECKED(vp, -1); CTR3(KTR_VFS, "%s: vp %p v_writecount decreased to %d", __func__, vp, vp->v_writecount); @@ -1525,7 +1530,7 @@ VM_OBJECT_WUNLOCK(obj); } -void +static void vnode_pager_update_writecount(vm_object_t object, vm_offset_t start, vm_offset_t end) { @@ -1537,15 +1542,15 @@ VM_OBJECT_WUNLOCK(object); return; } - old_wm = object->un_pager.vnp.writemappings; - object->un_pager.vnp.writemappings += (vm_ooffset_t)end - start; + old_wm = object->writemappings; + object->writemappings += (vm_ooffset_t)end - start; vp = object->handle; - if (old_wm == 0 && object->un_pager.vnp.writemappings != 0) { + if (old_wm == 0 && object->writemappings != 0) { ASSERT_VOP_LOCKED(vp, "v_writecount inc"); VOP_ADD_WRITECOUNT_CHECKED(vp, 1); CTR3(KTR_VFS, "%s: vp %p v_writecount increased to %d", __func__, vp, vp->v_writecount); - } else if (old_wm != 0 && object->un_pager.vnp.writemappings == 0) { + } else if (old_wm != 0 && object->writemappings == 0) { ASSERT_VOP_LOCKED(vp, "v_writecount dec"); VOP_ADD_WRITECOUNT_CHECKED(vp, -1); CTR3(KTR_VFS, "%s: vp %p v_writecount decreased to %d", @@ -1554,7 +1559,7 @@ VM_OBJECT_WUNLOCK(object); } -void +static void vnode_pager_release_writecount(vm_object_t object, vm_offset_t start, vm_offset_t end) { @@ -1578,8 +1583,8 @@ * zero. */ inc = end - start; - if (object->un_pager.vnp.writemappings != inc) { - object->un_pager.vnp.writemappings -= inc; + if (object->writemappings != inc) { + object->writemappings -= inc; VM_OBJECT_WUNLOCK(object); return; }