Index: sys/kern/vfs_subr.c =================================================================== --- sys/kern/vfs_subr.c +++ sys/kern/vfs_subr.c @@ -1663,6 +1663,65 @@ return (0); } +static void +freevnode(struct vnode *vp) +{ + struct bufobj *bo; + + /* + * The vnode has been marked for destruction, so free it. + * + * The vnode will be returned to the zone where it will + * normally remain until it is needed for another vnode. We + * need to cleanup (or verify that the cleanup has already + * been done) any residual data left from its current use + * so as not to contaminate the freshly allocated vnode. + */ + CTR2(KTR_VFS, "%s: destroying the vnode %p", __func__, vp); + atomic_subtract_long(&numvnodes, 1); + bo = &vp->v_bufobj; + VNASSERT((vp->v_iflag & VI_FREE) == 0, vp, + ("cleaned vnode still on the free list.")); + VNASSERT(vp->v_data == NULL, vp, ("cleaned vnode isn't")); + VNASSERT(vp->v_holdcnt == 0, vp, ("Non-zero hold count")); + VNASSERT(vp->v_usecount == 0, vp, ("Non-zero use count")); + VNASSERT(vp->v_writecount == 0, vp, ("Non-zero write count")); + VNASSERT(bo->bo_numoutput == 0, vp, ("Clean vnode has pending I/O's")); + VNASSERT(bo->bo_clean.bv_cnt == 0, vp, ("cleanbufcnt not 0")); + VNASSERT(pctrie_is_empty(&bo->bo_clean.bv_root), vp, + ("clean blk trie not empty")); + VNASSERT(bo->bo_dirty.bv_cnt == 0, vp, ("dirtybufcnt not 0")); + VNASSERT(pctrie_is_empty(&bo->bo_dirty.bv_root), vp, + ("dirty blk trie not empty")); + VNASSERT(TAILQ_EMPTY(&vp->v_cache_dst), vp, ("vp has namecache dst")); + VNASSERT(LIST_EMPTY(&vp->v_cache_src), vp, ("vp has namecache src")); + VNASSERT(vp->v_cache_dd == NULL, vp, ("vp has namecache for ..")); + VNASSERT(TAILQ_EMPTY(&vp->v_rl.rl_waiters), vp, + ("Dangling rangelock waiters")); + VI_UNLOCK(vp); +#ifdef MAC + mac_vnode_destroy(vp); +#endif + if (vp->v_pollinfo != NULL) { + destroy_vpollinfo(vp->v_pollinfo); + vp->v_pollinfo = NULL; + } +#ifdef INVARIANTS + /* XXX Elsewhere we detect an already freed vnode via NULL v_op. */ + vp->v_op = NULL; +#endif + vp->v_mountedhere = NULL; + vp->v_unpcb = NULL; + vp->v_rdev = NULL; + vp->v_fifoinfo = NULL; + vp->v_lasta = vp->v_clen = vp->v_cstart = vp->v_lastw = 0; + vp->v_irflag = 0; + vp->v_iflag = 0; + vp->v_vflag = 0; + bo->bo_flag = 0; + uma_zfree(vnode_zone, vp); +} + /* * Delete from old mount point vnode list, if on one. */ @@ -2707,7 +2766,7 @@ * usecount is permitted to transition 1->0 without the interlock because * vnode is kept live by holdcnt. */ -static enum vgetstate +static enum vgetstate __always_inline _vget_prep(struct vnode *vp, bool interlock) { enum vgetstate vs; @@ -2715,7 +2774,10 @@ if (refcount_acquire_if_not_zero(&vp->v_usecount)) { vs = VGET_USECOUNT; } else { - _vhold(vp, interlock); + if (interlock) + vholdl(vp); + else + vhold(vp); vs = VGET_HOLDCNT; } return (vs); @@ -3048,31 +3110,12 @@ /* * Increase the hold count and activate if this is the first reference. */ -void -_vhold(struct vnode *vp, bool locked) +static void +vhold_activate(struct vnode *vp) { struct mount *mp; - if (locked) - ASSERT_VI_LOCKED(vp, __func__); - else - ASSERT_VI_UNLOCKED(vp, __func__); - CTR2(KTR_VFS, "%s: vp %p", __func__, vp); - if (!locked) { - if (refcount_acquire_if_not_zero(&vp->v_holdcnt)) { - VNODE_REFCOUNT_FENCE_ACQ(); - VNASSERT((vp->v_iflag & VI_FREE) == 0, vp, - ("_vhold: vnode with holdcnt is free")); - return; - } - VI_LOCK(vp); - } - if ((vp->v_iflag & VI_FREE) == 0) { - refcount_acquire(&vp->v_holdcnt); - if (!locked) - VI_UNLOCK(vp); - return; - } + ASSERT_VI_LOCKED(vp, __func__); VNASSERT(vp->v_holdcnt == 0, vp, ("%s: wrong hold count", __func__)); VNASSERT(vp->v_op != NULL, vp, @@ -3082,7 +3125,7 @@ * and put it on the active list. */ VNASSERT(vp->v_mount != NULL, vp, - ("_vhold: vnode not on per mount vnode list")); + ("vhold: vnode not on per mount vnode list")); mp = vp->v_mount; mtx_lock(&mp->mnt_listmtx); if ((vp->v_mflag & VMP_TMPMNTFREELIST) != 0) { @@ -3103,8 +3146,36 @@ mp->mnt_activevnodelistsize++; mtx_unlock(&mp->mnt_listmtx); refcount_acquire(&vp->v_holdcnt); - if (!locked) - VI_UNLOCK(vp); +} + +void +vhold(struct vnode *vp) +{ + + ASSERT_VI_UNLOCKED(vp, __func__); + CTR2(KTR_VFS, "%s: vp %p", __func__, vp); + if (refcount_acquire_if_not_zero(&vp->v_holdcnt)) { + VNODE_REFCOUNT_FENCE_ACQ(); + VNASSERT((vp->v_iflag & VI_FREE) == 0, vp, + ("_vhold: vnode with holdcnt is free")); + return; + } + VI_LOCK(vp); + vholdl(vp); + VI_UNLOCK(vp); +} + +void +vholdl(struct vnode *vp) +{ + + ASSERT_VI_LOCKED(vp, __func__); + CTR2(KTR_VFS, "%s: vp %p", __func__, vp); + if ((vp->v_iflag & VI_FREE) == 0) { + refcount_acquire(&vp->v_holdcnt); + return; + } + vhold_activate(vp); } void @@ -3129,131 +3200,72 @@ * there is at least one resident non-cached page, the vnode cannot * leave the active list without the page cleanup done. */ -void -_vdrop(struct vnode *vp, bool locked) +static void +vdrop_deactivate(struct vnode *vp) { - struct bufobj *bo; struct mount *mp; - if (locked) - ASSERT_VI_LOCKED(vp, __func__); - else - ASSERT_VI_UNLOCKED(vp, __func__); - CTR2(KTR_VFS, "%s: vp %p", __func__, vp); - if (__predict_false((int)vp->v_holdcnt <= 0)) { - vn_printf(vp, "vdrop: holdcnt %d", vp->v_holdcnt); - panic("vdrop: wrong holdcnt"); - } - if (!locked) { - if (refcount_release_if_not_last(&vp->v_holdcnt)) - return; - VI_LOCK(vp); + /* + * Mark a vnode as free: remove it from its active list + * and put it up for recycling on the freelist. + */ + VNASSERT(vp->v_op != NULL, vp, + ("vdrop: vnode already reclaimed.")); + VNASSERT(!VN_IS_DOOMED(vp), vp, + ("vdrop: returning doomed vnode")); + VNASSERT((vp->v_iflag & VI_FREE) == 0, vp, + ("vdrop: vnode already free")); + VNASSERT(vp->v_holdcnt == 0, vp, + ("vdrop: freeing when we shouldn't")); + if ((vp->v_iflag & VI_OWEINACT) == 0) { + mp = vp->v_mount; + mtx_lock(&mp->mnt_listmtx); + vp->v_iflag &= ~VI_ACTIVE; + TAILQ_REMOVE(&mp->mnt_activevnodelist, vp, v_actfreelist); + mp->mnt_activevnodelistsize--; + TAILQ_INSERT_TAIL(&mp->mnt_tmpfreevnodelist, vp, v_actfreelist); + mp->mnt_tmpfreevnodelistsize++; + vp->v_iflag |= VI_FREE; + vp->v_mflag |= VMP_TMPMNTFREELIST; + VI_UNLOCK(vp); + if (mp->mnt_tmpfreevnodelistsize >= mnt_free_list_batch) + vnlru_return_batch_locked(mp); + mtx_unlock(&mp->mnt_listmtx); + } else { + VI_UNLOCK(vp); + counter_u64_add(free_owe_inact, 1); } - if (refcount_release(&vp->v_holdcnt) == 0) { +} + +void +vdrop(struct vnode *vp) +{ + + ASSERT_VI_UNLOCKED(vp, __func__); + CTR2(KTR_VFS, "%s: vp %p", __func__, vp); + if (refcount_release_if_not_last(&vp->v_holdcnt)) + return; + VI_LOCK(vp); + vdropl(vp); +} + +void +vdropl(struct vnode *vp) +{ + + ASSERT_VI_LOCKED(vp, __func__); + CTR2(KTR_VFS, "%s: vp %p", __func__, vp); + VNASSERT(VN_IS_DOOMED(vp) || (vp->v_iflag & VI_ACTIVE) != 0, vp, + ("vdrop: if vnode is not doomed it must be active")); + if (!refcount_release(&vp->v_holdcnt)) { VI_UNLOCK(vp); return; } - if (!VN_IS_DOOMED(vp)) { - /* - * Mark a vnode as free: remove it from its active list - * and put it up for recycling on the freelist. - */ - VNASSERT(vp->v_op != NULL, vp, - ("vdropl: vnode already reclaimed.")); - VNASSERT((vp->v_iflag & VI_FREE) == 0, vp, - ("vnode already free")); - VNASSERT(vp->v_holdcnt == 0, vp, - ("vdropl: freeing when we shouldn't")); - if ((vp->v_iflag & VI_OWEINACT) == 0) { - mp = vp->v_mount; - if (mp != NULL) { - mtx_lock(&mp->mnt_listmtx); - if (vp->v_iflag & VI_ACTIVE) { - vp->v_iflag &= ~VI_ACTIVE; - TAILQ_REMOVE(&mp->mnt_activevnodelist, - vp, v_actfreelist); - mp->mnt_activevnodelistsize--; - } - TAILQ_INSERT_TAIL(&mp->mnt_tmpfreevnodelist, - vp, v_actfreelist); - mp->mnt_tmpfreevnodelistsize++; - vp->v_iflag |= VI_FREE; - vp->v_mflag |= VMP_TMPMNTFREELIST; - VI_UNLOCK(vp); - if (mp->mnt_tmpfreevnodelistsize >= - mnt_free_list_batch) - vnlru_return_batch_locked(mp); - mtx_unlock(&mp->mnt_listmtx); - } else { - VNASSERT((vp->v_iflag & VI_ACTIVE) == 0, vp, - ("vdropl: active vnode not on per mount " - "vnode list")); - mtx_lock(&vnode_free_list_mtx); - TAILQ_INSERT_TAIL(&vnode_free_list, vp, - v_actfreelist); - freevnodes++; - vp->v_iflag |= VI_FREE; - VI_UNLOCK(vp); - mtx_unlock(&vnode_free_list_mtx); - } - } else { - VI_UNLOCK(vp); - counter_u64_add(free_owe_inact, 1); - } + if (VN_IS_DOOMED(vp)) { + freevnode(vp); return; } - /* - * The vnode has been marked for destruction, so free it. - * - * The vnode will be returned to the zone where it will - * normally remain until it is needed for another vnode. We - * need to cleanup (or verify that the cleanup has already - * been done) any residual data left from its current use - * so as not to contaminate the freshly allocated vnode. - */ - CTR2(KTR_VFS, "%s: destroying the vnode %p", __func__, vp); - atomic_subtract_long(&numvnodes, 1); - bo = &vp->v_bufobj; - VNASSERT((vp->v_iflag & VI_FREE) == 0, vp, - ("cleaned vnode still on the free list.")); - VNASSERT(vp->v_data == NULL, vp, ("cleaned vnode isn't")); - VNASSERT(vp->v_holdcnt == 0, vp, ("Non-zero hold count")); - VNASSERT(vp->v_usecount == 0, vp, ("Non-zero use count")); - VNASSERT(vp->v_writecount == 0, vp, ("Non-zero write count")); - VNASSERT(bo->bo_numoutput == 0, vp, ("Clean vnode has pending I/O's")); - VNASSERT(bo->bo_clean.bv_cnt == 0, vp, ("cleanbufcnt not 0")); - VNASSERT(pctrie_is_empty(&bo->bo_clean.bv_root), vp, - ("clean blk trie not empty")); - VNASSERT(bo->bo_dirty.bv_cnt == 0, vp, ("dirtybufcnt not 0")); - VNASSERT(pctrie_is_empty(&bo->bo_dirty.bv_root), vp, - ("dirty blk trie not empty")); - VNASSERT(TAILQ_EMPTY(&vp->v_cache_dst), vp, ("vp has namecache dst")); - VNASSERT(LIST_EMPTY(&vp->v_cache_src), vp, ("vp has namecache src")); - VNASSERT(vp->v_cache_dd == NULL, vp, ("vp has namecache for ..")); - VNASSERT(TAILQ_EMPTY(&vp->v_rl.rl_waiters), vp, - ("Dangling rangelock waiters")); - VI_UNLOCK(vp); -#ifdef MAC - mac_vnode_destroy(vp); -#endif - if (vp->v_pollinfo != NULL) { - destroy_vpollinfo(vp->v_pollinfo); - vp->v_pollinfo = NULL; - } -#ifdef INVARIANTS - /* XXX Elsewhere we detect an already freed vnode via NULL v_op. */ - vp->v_op = NULL; -#endif - vp->v_mountedhere = NULL; - vp->v_unpcb = NULL; - vp->v_rdev = NULL; - vp->v_fifoinfo = NULL; - vp->v_lasta = vp->v_clen = vp->v_cstart = vp->v_lastw = 0; - vp->v_irflag = 0; - vp->v_iflag = 0; - vp->v_vflag = 0; - bo->bo_flag = 0; - uma_zfree(vnode_zone, vp); + vdrop_deactivate(vp); } /* Index: sys/sys/vnode.h =================================================================== --- sys/sys/vnode.h +++ sys/sys/vnode.h @@ -653,17 +653,15 @@ struct ucred *cred, int *privused); void vattr_null(struct vattr *vap); int vcount(struct vnode *vp); -#define vdrop(vp) _vdrop((vp), 0) -#define vdropl(vp) _vdrop((vp), 1) -void _vdrop(struct vnode *, bool); +void vdrop(struct vnode *); +void vdropl(struct vnode *); int vflush(struct mount *mp, int rootrefs, int flags, struct thread *td); int vget(struct vnode *vp, int flags, struct thread *td); enum vgetstate vget_prep(struct vnode *vp); int vget_finish(struct vnode *vp, int flags, enum vgetstate vs); void vgone(struct vnode *vp); -#define vhold(vp) _vhold((vp), 0) -#define vholdl(vp) _vhold((vp), 1) -void _vhold(struct vnode *, bool); +void vhold(struct vnode *); +void vholdl(struct vnode *); void vholdnz(struct vnode *); void vinactive(struct vnode *, struct thread *); int vinvalbuf(struct vnode *vp, int save, int slpflag, int slptimeo);