Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/vfs_subr.c
Show First 20 Lines • Show All 101 Lines • ▼ Show 20 Lines | |||||
#endif | #endif | ||||
static void delmntque(struct vnode *vp); | static void delmntque(struct vnode *vp); | ||||
static int flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo, | static int flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo, | ||||
int slpflag, int slptimeo); | int slpflag, int slptimeo); | ||||
static void syncer_shutdown(void *arg, int howto); | static void syncer_shutdown(void *arg, int howto); | ||||
static int vtryrecycle(struct vnode *vp); | static int vtryrecycle(struct vnode *vp); | ||||
static void v_init_counters(struct vnode *); | static void v_init_counters(struct vnode *); | ||||
static void v_incr_usecount(struct vnode *); | |||||
static void v_incr_usecount_locked(struct vnode *); | |||||
static void v_incr_devcount(struct vnode *); | static void v_incr_devcount(struct vnode *); | ||||
static void v_decr_devcount(struct vnode *); | static void v_decr_devcount(struct vnode *); | ||||
static void vgonel(struct vnode *); | static void vgonel(struct vnode *); | ||||
static void vfs_knllock(void *arg); | static void vfs_knllock(void *arg); | ||||
static void vfs_knlunlock(void *arg); | static void vfs_knlunlock(void *arg); | ||||
static void vfs_knl_assert_locked(void *arg); | static void vfs_knl_assert_locked(void *arg); | ||||
static void vfs_knl_assert_unlocked(void *arg); | static void vfs_knl_assert_unlocked(void *arg); | ||||
static void vnlru_return_batches(struct vfsops *mnt_op); | static void vnlru_return_batches(struct vfsops *mnt_op); | ||||
▲ Show 20 Lines • Show All 2,507 Lines • ▼ Show 20 Lines | v_init_counters(struct vnode *vp) | ||||
VNASSERT(vp->v_type == VNON && vp->v_data == NULL && vp->v_iflag == 0, | VNASSERT(vp->v_type == VNON && vp->v_data == NULL && vp->v_iflag == 0, | ||||
vp, ("%s called for an initialized vnode", __FUNCTION__)); | vp, ("%s called for an initialized vnode", __FUNCTION__)); | ||||
ASSERT_VI_UNLOCKED(vp, __FUNCTION__); | ASSERT_VI_UNLOCKED(vp, __FUNCTION__); | ||||
refcount_init(&vp->v_holdcnt, 1); | refcount_init(&vp->v_holdcnt, 1); | ||||
refcount_init(&vp->v_usecount, 1); | refcount_init(&vp->v_usecount, 1); | ||||
} | } | ||||
static void | |||||
v_incr_usecount_locked(struct vnode *vp) | |||||
{ | |||||
ASSERT_VI_LOCKED(vp, __func__); | |||||
if ((vp->v_iflag & VI_OWEINACT) != 0) { | |||||
VNASSERT(vp->v_usecount == 0, vp, | |||||
("vnode with usecount and VI_OWEINACT set")); | |||||
vp->v_iflag &= ~VI_OWEINACT; | |||||
} | |||||
refcount_acquire(&vp->v_usecount); | |||||
v_incr_devcount(vp); | |||||
} | |||||
/* | /* | ||||
* Increment the use count on the vnode, taking care to reference | |||||
* the driver's usecount if this is a chardev. | |||||
*/ | |||||
static void | |||||
v_incr_usecount(struct vnode *vp) | |||||
{ | |||||
ASSERT_VI_UNLOCKED(vp, __func__); | |||||
CTR2(KTR_VFS, "%s: vp %p", __func__, vp); | |||||
if (vp->v_type != VCHR && | |||||
refcount_acquire_if_not_zero(&vp->v_usecount)) { | |||||
VNODE_REFCOUNT_FENCE_ACQ(); | |||||
VNASSERT((vp->v_iflag & VI_OWEINACT) == 0, vp, | |||||
("vnode with usecount and VI_OWEINACT set")); | |||||
} else { | |||||
VI_LOCK(vp); | |||||
v_incr_usecount_locked(vp); | |||||
VI_UNLOCK(vp); | |||||
} | |||||
} | |||||
/* | |||||
* Increment si_usecount of the associated device, if any. | * Increment si_usecount of the associated device, if any. | ||||
*/ | */ | ||||
static void | static void | ||||
v_incr_devcount(struct vnode *vp) | v_incr_devcount(struct vnode *vp) | ||||
{ | { | ||||
ASSERT_VI_LOCKED(vp, __FUNCTION__); | ASSERT_VI_LOCKED(vp, __FUNCTION__); | ||||
if (vp->v_type == VCHR && vp->v_rdev != NULL) { | if (vp->v_type == VCHR && vp->v_rdev != NULL) { | ||||
Show All 27 Lines | |||||
* | * | ||||
* Notes on lockless counter manipulation: | * Notes on lockless counter manipulation: | ||||
* _vhold, vputx and other routines make various decisions based | * _vhold, vputx and other routines make various decisions based | ||||
* on either holdcnt or usecount being 0. As long as either counter | * on either holdcnt or usecount being 0. As long as either counter | ||||
* is not transitioning 0->1 nor 1->0, the manipulation can be done | * is not transitioning 0->1 nor 1->0, the manipulation can be done | ||||
* with atomic operations. Otherwise the interlock is taken covering | * with atomic operations. Otherwise the interlock is taken covering | ||||
* both the atomic and additional actions. | * both the atomic and additional actions. | ||||
*/ | */ | ||||
static int | |||||
_vget_prep(struct vnode *vp, bool interlock) | |||||
{ | |||||
int rv; | |||||
if (__predict_true(vp->v_type != VCHR)) { | |||||
if (refcount_acquire_if_not_zero(&vp->v_usecount)) { | |||||
rv = LK_VNUSED; | |||||
} else { | |||||
_vhold(vp, interlock); | |||||
rv = LK_VNHELD; | |||||
} | |||||
} else { | |||||
if (!interlock) { | |||||
VI_LOCK(vp); | |||||
interlock = true; | |||||
} | |||||
if (vp->v_usecount == 0) { | |||||
vholdl(vp); | |||||
rv = LK_VNHELD; | |||||
} else { | |||||
v_incr_devcount(vp); | |||||
refcount_acquire(&vp->v_usecount); | |||||
rv = LK_VNUSED; | |||||
} | |||||
} | |||||
if (interlock) | |||||
VI_UNLOCK(vp); | |||||
return (rv); | |||||
} | |||||
int | int | ||||
vget_prep(struct vnode *vp) | |||||
{ | |||||
return (_vget_prep(vp, false)); | |||||
} | |||||
int | |||||
vget(struct vnode *vp, int flags, struct thread *td) | vget(struct vnode *vp, int flags, struct thread *td) | ||||
{ | { | ||||
MPASS(td == curthread); | |||||
flags |= _vget_prep(vp, (flags & LK_INTERLOCK) != 0); | |||||
return (vget_finish(vp, flags & ~LK_INTERLOCK)); | |||||
} | |||||
kib: Why do you need LK_VNUSED ? You can pass additional arg to vget_finish() easily, it seems. | |||||
Done Inline ActionsI noted it's an option (and in fact preferred by me), but I went with this because LK_VNHELD was already there. More than happy to change it and get rid of LK_VNHELD. mjg: I noted it's an option (and in fact preferred by me), but I went with this because LK_VNHELD… | |||||
int | |||||
vget_finish(struct vnode *vp, int flags) | |||||
{ | |||||
int error, oweinact; | int error, oweinact; | ||||
VNASSERT((flags & LK_TYPE_MASK) != 0, vp, | VNASSERT((flags & LK_TYPE_MASK) != 0, vp, | ||||
("vget: invalid lock operation")); | ("%s: invalid lock operation", __func__)); | ||||
KASSERT(((flags & LK_INTERLOCK) == 0), | |||||
if ((flags & LK_INTERLOCK) != 0) | ("%s: LK_INTERLOCK passed\n", __func__)); | ||||
ASSERT_VI_LOCKED(vp, __func__); | |||||
else | |||||
ASSERT_VI_UNLOCKED(vp, __func__); | ASSERT_VI_UNLOCKED(vp, __func__); | ||||
if ((flags & LK_VNHELD) != 0) | VNASSERT(vp->v_holdcnt > 0, vp, ("%s: vnode not held", __func__)); | ||||
VNASSERT((vp->v_holdcnt > 0), vp, | if (flags & LK_VNUSED) { | ||||
("vget: LK_VNHELD passed but vnode not held")); | VNASSERT(vp->v_usecount > 0, vp, | ||||
("%s: vnode without usecount when LK_VNUSED was set", | |||||
__func__)); | |||||
} | |||||
CTR3(KTR_VFS, "%s: vp %p with flags %d", __func__, vp, flags); | |||||
if ((flags & LK_VNHELD) == 0) | |||||
_vhold(vp, (flags & LK_INTERLOCK) != 0); | |||||
if ((error = vn_lock(vp, flags)) != 0) { | if ((error = vn_lock(vp, flags)) != 0) { | ||||
if (flags & LK_VNUSED) | |||||
vrele(vp); | |||||
else | |||||
vdrop(vp); | vdrop(vp); | ||||
CTR2(KTR_VFS, "%s: impossible to lock vnode %p", __func__, | CTR2(KTR_VFS, "%s: impossible to lock vnode %p", __func__, | ||||
vp); | vp); | ||||
return (error); | return (error); | ||||
} | } | ||||
if (vp->v_iflag & VI_DOOMED && (flags & LK_RETRY) == 0) | |||||
panic("vget: vn_lock failed to return ENOENT\n"); | if (flags & LK_VNUSED) { | ||||
VNASSERT((vp->v_iflag & VI_OWEINACT) == 0, vp, | |||||
("%s: vnode with usecount and VI_OWEINACT set", __func__)); | |||||
return (0); | |||||
} | |||||
/* | /* | ||||
* We hold the vnode. If the usecount is 0 it will be utilized to keep | |||||
* the vnode around. Otherwise someone else lended their hold count and | |||||
* we have to drop ours. | |||||
*/ | |||||
if (vp->v_type != VCHR && | |||||
refcount_acquire_if_not_zero(&vp->v_usecount)) { | |||||
#ifdef INVARIANTS | |||||
int old = atomic_fetchadd_int(&vp->v_holdcnt, -1) - 1; | |||||
VNASSERT(old > 0, vp, ("%s: wrong hold count", __func__)); | |||||
#else | |||||
refcount_release(&vp->v_holdcnt); | |||||
#endif | |||||
VNODE_REFCOUNT_FENCE_ACQ(); | |||||
VNASSERT((vp->v_iflag & VI_OWEINACT) == 0, vp, | |||||
("%s: vnode with usecount and VI_OWEINACT set", __func__)); | |||||
return (0); | |||||
} | |||||
/* | |||||
* We don't guarantee that any particular close will | * We don't guarantee that any particular close will | ||||
* trigger inactive processing so just make a best effort | * trigger inactive processing so just make a best effort | ||||
* here at preventing a reference to a removed file. If | * here at preventing a reference to a removed file. If | ||||
* we don't succeed no harm is done. | * we don't succeed no harm is done. | ||||
* | * | ||||
* Upgrade our holdcnt to a usecount. | * Upgrade our holdcnt to a usecount. | ||||
*/ | */ | ||||
if (vp->v_type == VCHR || | |||||
!refcount_acquire_if_not_zero(&vp->v_usecount)) { | |||||
VI_LOCK(vp); | VI_LOCK(vp); | ||||
if ((vp->v_iflag & VI_OWEINACT) == 0) { | if ((vp->v_iflag & VI_OWEINACT) == 0) { | ||||
oweinact = 0; | oweinact = 0; | ||||
} else { | } else { | ||||
oweinact = 1; | oweinact = 1; | ||||
vp->v_iflag &= ~VI_OWEINACT; | vp->v_iflag &= ~VI_OWEINACT; | ||||
VNODE_REFCOUNT_FENCE_REL(); | VNODE_REFCOUNT_FENCE_REL(); | ||||
} | } | ||||
refcount_acquire(&vp->v_usecount); | if (vp->v_usecount > 0) | ||||
refcount_release(&vp->v_holdcnt); | |||||
v_incr_devcount(vp); | v_incr_devcount(vp); | ||||
refcount_acquire(&vp->v_usecount); | |||||
if (oweinact && VOP_ISLOCKED(vp) == LK_EXCLUSIVE && | if (oweinact && VOP_ISLOCKED(vp) == LK_EXCLUSIVE && | ||||
(flags & LK_NOWAIT) == 0) | (flags & LK_NOWAIT) == 0) | ||||
vinactive(vp, td); | vinactive(vp, curthread); | ||||
VI_UNLOCK(vp); | VI_UNLOCK(vp); | ||||
} | |||||
return (0); | return (0); | ||||
} | } | ||||
/* | /* | ||||
* Increase the reference (use) and hold count of a vnode. | * Increase the reference (use) and hold count of a vnode. | ||||
* This will also remove the vnode from the free list if it is presently free. | * This will also remove the vnode from the free list if it is presently free. | ||||
*/ | */ | ||||
void | void | ||||
vref(struct vnode *vp) | vref(struct vnode *vp) | ||||
{ | { | ||||
ASSERT_VI_UNLOCKED(vp, __func__); | ASSERT_VI_UNLOCKED(vp, __func__); | ||||
CTR2(KTR_VFS, "%s: vp %p", __func__, vp); | CTR2(KTR_VFS, "%s: vp %p", __func__, vp); | ||||
_vhold(vp, false); | if (vp->v_type != VCHR && | ||||
v_incr_usecount(vp); | refcount_acquire_if_not_zero(&vp->v_usecount)) { | ||||
VNODE_REFCOUNT_FENCE_ACQ(); | |||||
VNASSERT(vp->v_holdcnt > 0, vp, | |||||
("%s: active vnode not held", __func__)); | |||||
VNASSERT((vp->v_iflag & VI_OWEINACT) == 0, vp, | |||||
("%s: vnode with usecount and VI_OWEINACT set", __func__)); | |||||
return; | |||||
} | } | ||||
VI_LOCK(vp); | |||||
vrefl(vp); | |||||
VI_UNLOCK(vp); | |||||
} | |||||
void | void | ||||
vrefl(struct vnode *vp) | vrefl(struct vnode *vp) | ||||
{ | { | ||||
ASSERT_VI_LOCKED(vp, __func__); | ASSERT_VI_LOCKED(vp, __func__); | ||||
CTR2(KTR_VFS, "%s: vp %p", __func__, vp); | CTR2(KTR_VFS, "%s: vp %p", __func__, vp); | ||||
_vhold(vp, true); | if (vp->v_usecount == 0) | ||||
v_incr_usecount_locked(vp); | vholdl(vp); | ||||
if ((vp->v_iflag & VI_OWEINACT) != 0) { | |||||
vp->v_iflag &= ~VI_OWEINACT; | |||||
VNODE_REFCOUNT_FENCE_REL(); | |||||
Not Done Inline ActionsWhy rel ? Even, why this barrier is needed ? kib: Why rel ? Even, why this barrier is needed ? | |||||
Done Inline ActionsThere were assertions failing on powerpc where the cpu would reorder flags vs refcount manipulation. mjg: There were assertions failing on powerpc where the cpu would reorder flags vs refcount… | |||||
} | } | ||||
v_incr_devcount(vp); | |||||
refcount_acquire(&vp->v_usecount); | |||||
} | |||||
void | void | ||||
vrefact(struct vnode *vp) | vrefact(struct vnode *vp) | ||||
{ | { | ||||
CTR2(KTR_VFS, "%s: vp %p", __func__, vp); | CTR2(KTR_VFS, "%s: vp %p", __func__, vp); | ||||
if (__predict_false(vp->v_type == VCHR)) { | if (__predict_false(vp->v_type == VCHR)) { | ||||
VNASSERT(vp->v_holdcnt > 0 && vp->v_usecount > 0, vp, | VNASSERT(vp->v_holdcnt > 0 && vp->v_usecount > 0, vp, | ||||
("%s: wrong ref counts", __func__)); | ("%s: wrong ref counts", __func__)); | ||||
vref(vp); | vref(vp); | ||||
return; | return; | ||||
} | } | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
int old = atomic_fetchadd_int(&vp->v_holdcnt, 1); | int old = atomic_fetchadd_int(&vp->v_usecount, 1); | ||||
VNASSERT(old > 0, vp, ("%s: wrong hold count", __func__)); | |||||
old = atomic_fetchadd_int(&vp->v_usecount, 1); | |||||
VNASSERT(old > 0, vp, ("%s: wrong use count", __func__)); | VNASSERT(old > 0, vp, ("%s: wrong use count", __func__)); | ||||
#else | #else | ||||
refcount_acquire(&vp->v_holdcnt); | |||||
refcount_acquire(&vp->v_usecount); | refcount_acquire(&vp->v_usecount); | ||||
#endif | #endif | ||||
} | } | ||||
/* | /* | ||||
* Return reference count of a vnode. | * Return reference count of a vnode. | ||||
* | * | ||||
* The results of this call are only guaranteed when some mechanism is used to | * The results of this call are only guaranteed when some mechanism is used to | ||||
Show All 28 Lines | if (func == VPUTX_VUNREF) | ||||
ASSERT_VOP_LOCKED(vp, "vunref"); | ASSERT_VOP_LOCKED(vp, "vunref"); | ||||
else if (func == VPUTX_VPUT) | else if (func == VPUTX_VPUT) | ||||
ASSERT_VOP_LOCKED(vp, "vput"); | ASSERT_VOP_LOCKED(vp, "vput"); | ||||
else | else | ||||
KASSERT(func == VPUTX_VRELE, ("vputx: wrong func")); | KASSERT(func == VPUTX_VRELE, ("vputx: wrong func")); | ||||
ASSERT_VI_UNLOCKED(vp, __func__); | ASSERT_VI_UNLOCKED(vp, __func__); | ||||
VNASSERT(vp->v_holdcnt > 0 && vp->v_usecount > 0, vp, | VNASSERT(vp->v_holdcnt > 0 && vp->v_usecount > 0, vp, | ||||
("%s: wrong ref counts", __func__)); | ("%s: wrong ref counts", __func__)); | ||||
CTR2(KTR_VFS, "%s: vp %p", __func__, vp); | CTR2(KTR_VFS, "%s: vp %p", __func__, vp); | ||||
if (vp->v_type != VCHR && | |||||
refcount_release_if_not_last(&vp->v_usecount)) { | |||||
if (func == VPUTX_VPUT) | if (func == VPUTX_VPUT) | ||||
VOP_UNLOCK(vp, 0); | VOP_UNLOCK(vp, 0); | ||||
vdrop(vp); | |||||
if (vp->v_type != VCHR && | |||||
refcount_release_if_not_last(&vp->v_usecount)) | |||||
return; | return; | ||||
} | |||||
VI_LOCK(vp); | VI_LOCK(vp); | ||||
/* | /* | ||||
* We want to hold the vnode until the inactive finishes to | * We want to hold the vnode until the inactive finishes to | ||||
* prevent vgone() races. We drop the use count here and the | * prevent vgone() races. We drop the use count here and the | ||||
* hold count below when we're done. | * hold count below when we're done. | ||||
*/ | */ | ||||
if (!refcount_release(&vp->v_usecount) || | |||||
(vp->v_iflag & VI_DOINGINACT)) { | |||||
if (func == VPUTX_VPUT) | |||||
VOP_UNLOCK(vp, 0); | |||||
v_decr_devcount(vp); | v_decr_devcount(vp); | ||||
if (!refcount_release(&vp->v_usecount)) { | |||||
VI_UNLOCK(vp); | |||||
return; | |||||
} | |||||
if (vp->v_iflag & VI_DOINGINACT) { | |||||
vdropl(vp); | vdropl(vp); | ||||
return; | return; | ||||
} | } | ||||
v_decr_devcount(vp); | |||||
error = 0; | error = 0; | ||||
if (vp->v_usecount != 0) { | if (vp->v_usecount != 0) { | ||||
vn_printf(vp, "vputx: usecount not zero for vnode "); | vn_printf(vp, "vputx: usecount not zero for vnode "); | ||||
panic("vputx: usecount not zero"); | panic("vputx: usecount not zero"); | ||||
} | } | ||||
CTR2(KTR_VFS, "%s: return vnode %p to the freelist", __func__, vp); | CTR2(KTR_VFS, "%s: return vnode %p to the freelist", __func__, vp); | ||||
/* | /* | ||||
* Check if the fs wants to perform inactive processing. Note we | * Check if the fs wants to perform inactive processing. Note we | ||||
* may be only holding the interlock, in which case it is possible | * may be only holding the interlock, in which case it is possible | ||||
* someone else called vgone on the vnode and ->v_data is now NULL. | * someone else called vgone on the vnode and ->v_data is now NULL. | ||||
* Since vgone performs inactive on its own there is nothing to do | * Since vgone performs inactive on its own there is nothing to do | ||||
* here but to drop our hold count. | * here but to drop our hold count. | ||||
*/ | */ | ||||
if (__predict_false(vp->v_iflag & VI_DOOMED) || | if (__predict_false(vp->v_iflag & VI_DOOMED) || | ||||
VOP_NEED_INACTIVE(vp) == 0) { | VOP_NEED_INACTIVE(vp) == 0) { | ||||
if (func == VPUTX_VPUT) | |||||
VOP_UNLOCK(vp, 0); | |||||
vdropl(vp); | vdropl(vp); | ||||
return; | return; | ||||
} | } | ||||
/* | /* | ||||
* We must call VOP_INACTIVE with the node locked. Mark | * We must call VOP_INACTIVE with the node locked. Mark | ||||
* as VI_DOINGINACT to avoid recursion. | * as VI_DOINGINACT to avoid recursion. | ||||
*/ | */ | ||||
vp->v_iflag |= VI_OWEINACT; | vp->v_iflag |= VI_OWEINACT; | ||||
switch (func) { | switch (func) { | ||||
case VPUTX_VRELE: | case VPUTX_VRELE: | ||||
error = vn_lock(vp, LK_EXCLUSIVE | LK_INTERLOCK); | error = vn_lock(vp, LK_EXCLUSIVE | LK_INTERLOCK); | ||||
VI_LOCK(vp); | VI_LOCK(vp); | ||||
break; | break; | ||||
case VPUTX_VPUT: | case VPUTX_VPUT: | ||||
if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) { | error = VOP_LOCK(vp, LK_EXCLUSIVE | LK_INTERLOCK | LK_NOWAIT); | ||||
error = VOP_LOCK(vp, LK_UPGRADE | LK_INTERLOCK | | |||||
LK_NOWAIT); | |||||
VI_LOCK(vp); | VI_LOCK(vp); | ||||
} | |||||
break; | break; | ||||
case VPUTX_VUNREF: | case VPUTX_VUNREF: | ||||
if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) { | if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) { | ||||
error = VOP_LOCK(vp, LK_TRYUPGRADE | LK_INTERLOCK); | error = VOP_LOCK(vp, LK_TRYUPGRADE | LK_INTERLOCK); | ||||
VI_LOCK(vp); | VI_LOCK(vp); | ||||
} | } | ||||
break; | break; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 2,850 Lines • Show Last 20 Lines |
Why do you need LK_VNUSED ? You can pass additional arg to vget_finish() easily, it seems.