Differential D25612 Diff 75672 head/sys/kern/vfs_subr.c

Changeset View

Standalone View

head/sys/kern/vfs_subr.c

Show First 20 Lines • Show All 102 Lines • ▼ Show 20 Lines
#endif		#endif

static void delmntque(struct vnode *vp);		static void delmntque(struct vnode *vp);
static int flushbuflist(struct bufv bufv, int flags, struct bufobj bo,		static int flushbuflist(struct bufv bufv, int flags, struct bufobj bo,
int slpflag, int slptimeo);		int slpflag, int slptimeo);
static void syncer_shutdown(void *arg, int howto);		static void syncer_shutdown(void *arg, int howto);
static int vtryrecycle(struct vnode *vp);		static int vtryrecycle(struct vnode *vp);
static void v_init_counters(struct vnode *);		static void v_init_counters(struct vnode *);
static void v_incr_devcount(struct vnode *);
static void v_decr_devcount(struct vnode *);
static void vgonel(struct vnode *);		static void vgonel(struct vnode *);
static void vfs_knllock(void *arg);		static void vfs_knllock(void *arg);
static void vfs_knlunlock(void *arg);		static void vfs_knlunlock(void *arg);
static void vfs_knl_assert_locked(void *arg);		static void vfs_knl_assert_locked(void *arg);
static void vfs_knl_assert_unlocked(void *arg);		static void vfs_knl_assert_unlocked(void *arg);
static void destroy_vpollinfo(struct vpollinfo *vi);		static void destroy_vpollinfo(struct vpollinfo *vi);
static int v_inval_buf_range_locked(struct vnode vp, struct bufobj bo,		static int v_inval_buf_range_locked(struct vnode vp, struct bufobj bo,
daddr_t startlbn, daddr_t endlbn);		daddr_t startlbn, daddr_t endlbn);
▲ Show 20 Lines • Show All 2,668 Lines • ▼ Show 20 Lines	VNASSERT(vp->v_type == VNON && vp->v_data == NULL && vp->v_iflag == 0,
vp, ("%s called for an initialized vnode", __FUNCTION__));		vp, ("%s called for an initialized vnode", __FUNCTION__));
ASSERT_VI_UNLOCKED(vp, __FUNCTION__);		ASSERT_VI_UNLOCKED(vp, __FUNCTION__);

refcount_init(&vp->v_holdcnt, 1);		refcount_init(&vp->v_holdcnt, 1);
refcount_init(&vp->v_usecount, 1);		refcount_init(&vp->v_usecount, 1);
}		}

/*		/*
* Increment si_usecount of the associated device, if any.
*/
static void
v_incr_devcount(struct vnode *vp)
{

ASSERT_VI_LOCKED(vp, __FUNCTION__);
if (vp->v_type == VCHR && vp->v_rdev != NULL) {
dev_lock();
vp->v_rdev->si_usecount++;
dev_unlock();
}
}

/*
* Decrement si_usecount of the associated device, if any.
*
* The caller is required to hold the interlock when transitioning a VCHR use
* count to zero. This prevents a race with devfs_reclaim_vchr() that would
* leak a si_usecount reference. The vnode lock will also prevent this race
* if it is held while dropping the last ref.
*
* The race is:
*
* CPU1 CPU2
* devfs_reclaim_vchr
* make v_usecount == 0
* VI_LOCK
* sees v_usecount == 0, no updates
* vp->v_rdev = NULL;
* ...
* VI_UNLOCK
* VI_LOCK
* v_decr_devcount
* sees v_rdev == NULL, no updates
*
* In this scenario si_devcount decrement is not performed.
*/
static void
v_decr_devcount(struct vnode *vp)
{

ASSERT_VOP_LOCKED(vp, __func__);
ASSERT_VI_LOCKED(vp, __FUNCTION__);
if (vp->v_type == VCHR && vp->v_rdev != NULL) {
dev_lock();
VNPASS(vp->v_rdev->si_usecount > 0, vp);
vp->v_rdev->si_usecount--;
dev_unlock();
}
}

/*
* Grab a particular vnode from the free list, increment its		* Grab a particular vnode from the free list, increment its
* reference count and lock it. VIRF_DOOMED is set if the vnode		* reference count and lock it. VIRF_DOOMED is set if the vnode
* is being destroyed. Only callers who specify LK_RETRY will		* is being destroyed. Only callers who specify LK_RETRY will
* see doomed vnodes. If inactive processing was delayed in		* see doomed vnodes. If inactive processing was delayed in
* vput try to do it here.		* vput try to do it here.
*		*
* usecount is manipulated using atomics without holding any locks.		* usecount is manipulated using atomics without holding any locks.
*		*
▲ Show 20 Lines • Show All 58 Lines • ▼ Show 20 Lines	vget(struct vnode vp, int flags, struct thread td)
enum vgetstate vs;		enum vgetstate vs;

MPASS(td == curthread);		MPASS(td == curthread);

vs = vget_prep(vp);		vs = vget_prep(vp);
return (vget_finish(vp, flags, vs));		return (vget_finish(vp, flags, vs));
}		}

static void __noinline
vget_finish_vchr(struct vnode *vp)
{

VNASSERT(vp->v_type == VCHR, vp, ("type != VCHR)"));

/*
* See the comment in vget_finish before usecount bump.
*/
if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
#ifdef INVARIANTS
int old = atomic_fetchadd_int(&vp->v_holdcnt, -1);
VNASSERT(old > 0, vp, ("%s: wrong hold count %d", __func__, old));
#else
refcount_release(&vp->v_holdcnt);
#endif
return;
}

VI_LOCK(vp);
if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
#ifdef INVARIANTS
int old = atomic_fetchadd_int(&vp->v_holdcnt, -1);
VNASSERT(old > 1, vp, ("%s: wrong hold count %d", __func__, old));
#else
refcount_release(&vp->v_holdcnt);
#endif
VI_UNLOCK(vp);
return;
}
v_incr_devcount(vp);
refcount_acquire(&vp->v_usecount);
VI_UNLOCK(vp);
}

int		int
vget_finish(struct vnode *vp, int flags, enum vgetstate vs)		vget_finish(struct vnode *vp, int flags, enum vgetstate vs)
{		{
int error;		int error;

if ((flags & LK_INTERLOCK) != 0)		if ((flags & LK_INTERLOCK) != 0)
ASSERT_VI_LOCKED(vp, __func__);		ASSERT_VI_LOCKED(vp, __func__);
else		else
Show All 21 Lines	vget_finish_ref(struct vnode *vp, enum vgetstate vs)

VNPASS(vs == VGET_HOLDCNT \|\| vs == VGET_USECOUNT, vp);		VNPASS(vs == VGET_HOLDCNT \|\| vs == VGET_USECOUNT, vp);
VNPASS(vp->v_holdcnt > 0, vp);		VNPASS(vp->v_holdcnt > 0, vp);
VNPASS(vs == VGET_HOLDCNT \|\| vp->v_usecount > 0, vp);		VNPASS(vs == VGET_HOLDCNT \|\| vp->v_usecount > 0, vp);

if (vs == VGET_USECOUNT)		if (vs == VGET_USECOUNT)
return;		return;

if (__predict_false(vp->v_type == VCHR)) {
vget_finish_vchr(vp);
return;
}

/*		/*
* We hold the vnode. If the usecount is 0 it will be utilized to keep		* We hold the vnode. If the usecount is 0 it will be utilized to keep
* the vnode around. Otherwise someone else lended their hold count and		* the vnode around. Otherwise someone else lended their hold count and
* we have to drop ours.		* we have to drop ours.
*/		*/
old = atomic_fetchadd_int(&vp->v_usecount, 1);		old = atomic_fetchadd_int(&vp->v_usecount, 1);
VNASSERT(old >= 0, vp, ("%s: wrong use count %d", __func__, old));		VNASSERT(old >= 0, vp, ("%s: wrong use count %d", __func__, old));
if (old != 0) {		if (old != 0) {
#ifdef INVARIANTS		#ifdef INVARIANTS
old = atomic_fetchadd_int(&vp->v_holdcnt, -1);		old = atomic_fetchadd_int(&vp->v_holdcnt, -1);
VNASSERT(old > 1, vp, ("%s: wrong hold count %d", __func__, old));		VNASSERT(old > 1, vp, ("%s: wrong hold count %d", __func__, old));
#else		#else
refcount_release(&vp->v_holdcnt);		refcount_release(&vp->v_holdcnt);
#endif		#endif
}		}
}		}

/*		/*
* Increase the reference (use) and hold count of a vnode.		* Increase the reference (use) and hold count of a vnode.
* This will also remove the vnode from the free list if it is presently free.		* This will also remove the vnode from the free list if it is presently free.
*/		*/
static void __noinline
vref_vchr(struct vnode *vp, bool interlock)
{

/*
* See the comment in vget_finish before usecount bump.
*/
if (!interlock) {
if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
VNODE_REFCOUNT_FENCE_ACQ();
VNASSERT(vp->v_holdcnt > 0, vp,
("%s: active vnode not held", __func__));
return;
}
VI_LOCK(vp);
/*
* By the time we get here the vnode might have been doomed, at
* which point the 0->1 use count transition is no longer
* protected by the interlock. Since it can't bounce back to
* VCHR and requires vref semantics, punt it back
*/
if (__predict_false(vp->v_type == VBAD)) {
VI_UNLOCK(vp);
vref(vp);
return;
}
}
VNASSERT(vp->v_type == VCHR, vp, ("type != VCHR)"));
if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
VNODE_REFCOUNT_FENCE_ACQ();
VNASSERT(vp->v_holdcnt > 0, vp,
("%s: active vnode not held", __func__));
if (!interlock)
VI_UNLOCK(vp);
return;
}
vhold(vp);
v_incr_devcount(vp);
refcount_acquire(&vp->v_usecount);
if (!interlock)
VI_UNLOCK(vp);
return;
}

void		void
vref(struct vnode *vp)		vref(struct vnode *vp)
{		{
int old;		int old;

CTR2(KTR_VFS, "%s: vp %p", __func__, vp);		CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
if (__predict_false(vp->v_type == VCHR)) {
vref_vchr(vp, false);
return;
}

if (refcount_acquire_if_not_zero(&vp->v_usecount)) {		if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
VNODE_REFCOUNT_FENCE_ACQ();		VNODE_REFCOUNT_FENCE_ACQ();
VNASSERT(vp->v_holdcnt > 0, vp,		VNASSERT(vp->v_holdcnt > 0, vp,
("%s: active vnode not held", __func__));		("%s: active vnode not held", __func__));
return;		return;
}		}
vhold(vp);		vhold(vp);
/*		/*
Show All 12 Lines
}		}

void		void
vrefl(struct vnode *vp)		vrefl(struct vnode *vp)
{		{

ASSERT_VI_LOCKED(vp, __func__);		ASSERT_VI_LOCKED(vp, __func__);
CTR2(KTR_VFS, "%s: vp %p", __func__, vp);		CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
if (__predict_false(vp->v_type == VCHR)) {
vref_vchr(vp, true);
return;
}
vref(vp);		vref(vp);
}		}

void		void
vrefact(struct vnode *vp)		vrefact(struct vnode *vp)
{		{

CTR2(KTR_VFS, "%s: vp %p", __func__, vp);		CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
▲ Show 20 Lines • Show All 124 Lines • ▼ Show 20 Lines
enum vput_op { VRELE, VPUT, VUNREF };		enum vput_op { VRELE, VPUT, VUNREF };

/*		/*
* Handle ->v_usecount transitioning to 0.		* Handle ->v_usecount transitioning to 0.
*		*
* By releasing the last usecount we take ownership of the hold count which		* By releasing the last usecount we take ownership of the hold count which
* provides liveness of the vnode, meaning we have to vdrop.		* provides liveness of the vnode, meaning we have to vdrop.
*		*
* If the vnode is of type VCHR we may need to decrement si_usecount, see
* v_decr_devcount for details.
*
* For all vnodes we may need to perform inactive processing. It requires an		* For all vnodes we may need to perform inactive processing. It requires an
* exclusive lock on the vnode, while it is legal to call here with only a		* exclusive lock on the vnode, while it is legal to call here with only a
* shared lock (or no locks). If locking the vnode in an expected manner fails,		* shared lock (or no locks). If locking the vnode in an expected manner fails,
* inactive processing gets deferred to the syncer.		* inactive processing gets deferred to the syncer.
*		*
* XXX Some filesystems pass in an exclusively locked vnode and strongly depend		* XXX Some filesystems pass in an exclusively locked vnode and strongly depend
* on the lock being held all the way until VOP_INACTIVE. This in particular		* on the lock being held all the way until VOP_INACTIVE. This in particular
* happens with UFS which adds half-constructed vnodes to the hash, where they		* happens with UFS which adds half-constructed vnodes to the hash, where they
* can be found by other code.		* can be found by other code.
*/		*/
static void		static void
vput_final(struct vnode *vp, enum vput_op func)		vput_final(struct vnode *vp, enum vput_op func)
{		{
int error;		int error;
bool want_unlock;		bool want_unlock;

CTR2(KTR_VFS, "%s: vp %p", __func__, vp);		CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
VNPASS(vp->v_holdcnt > 0, vp);		VNPASS(vp->v_holdcnt > 0, vp);

VI_LOCK(vp);		VI_LOCK(vp);
if (__predict_false(vp->v_type == VCHR && func != VRELE))
v_decr_devcount(vp);

/*		/*
* By the time we got here someone else might have transitioned		* By the time we got here someone else might have transitioned
* the count back to > 0.		* the count back to > 0.
*/		*/
if (vp->v_usecount > 0)		if (vp->v_usecount > 0)
goto out;		goto out;

▲ Show 20 Lines • Show All 71 Lines • ▼ Show 20 Lines
}		}

/*		/*
* Decrement ->v_usecount for a vnode.		* Decrement ->v_usecount for a vnode.
*		*
* Releasing the last use count requires additional processing, see vput_final		* Releasing the last use count requires additional processing, see vput_final
* above for details.		* above for details.
*		*
* Note that releasing use count without the vnode lock requires special casing
* for VCHR, see v_decr_devcount for details.
*
* Comment above each variant denotes lock state on entry and exit.		* Comment above each variant denotes lock state on entry and exit.
*/		*/

static void __noinline
vrele_vchr(struct vnode *vp)
{

if (refcount_release_if_not_last(&vp->v_usecount))
return;
VI_LOCK(vp);
if (!refcount_release(&vp->v_usecount)) {
VI_UNLOCK(vp);
return;
}
v_decr_devcount(vp);
VI_UNLOCK(vp);
vput_final(vp, VRELE);
}

/*		/*
* in: any		* in: any
* out: same as passed in		* out: same as passed in
*/		*/
void		void
vrele(struct vnode *vp)		vrele(struct vnode *vp)
{		{

ASSERT_VI_UNLOCKED(vp, __func__);		ASSERT_VI_UNLOCKED(vp, __func__);
if (__predict_false(vp->v_type == VCHR)) {
vrele_vchr(vp);
return;
}
if (!refcount_release(&vp->v_usecount))		if (!refcount_release(&vp->v_usecount))
return;		return;
vput_final(vp, VRELE);		vput_final(vp, VRELE);
}		}

/*		/*
* in: locked		* in: locked
* out: unlocked		* out: unlocked
▲ Show 20 Lines • Show All 732 Lines • ▼ Show 20 Lines	vgonel(struct vnode *vp)
/*		/*
* Done with purge, reset to the standard lock and invalidate		* Done with purge, reset to the standard lock and invalidate
* the vnode.		* the vnode.
*/		*/
VI_LOCK(vp);		VI_LOCK(vp);
vp->v_vnlock = &vp->v_lock;		vp->v_vnlock = &vp->v_lock;
vp->v_op = &dead_vnodeops;		vp->v_op = &dead_vnodeops;
vp->v_type = VBAD;		vp->v_type = VBAD;
}

/*
* Calculate the total number of references to a special device.
*/
int
vcount(struct vnode *vp)
{
int count;

dev_lock();
count = vp->v_rdev->si_usecount;
dev_unlock();
return (count);
}		}

/*		/*
* Print out a description of a vnode.		* Print out a description of a vnode.
*/		*/
static const char * const typename[] =		static const char * const typename[] =
{"VNON", "VREG", "VDIR", "VBLK", "VCHR", "VLNK", "VSOCK", "VFIFO", "VBAD",		{"VNON", "VREG", "VDIR", "VBLK", "VCHR", "VLNK", "VSOCK", "VFIFO", "VBAD",
"VMARKER"};		"VMARKER"};
▲ Show 20 Lines • Show All 2,757 Lines • Show Last 20 Lines