Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/vfs_vnops.c
Show First 20 Lines • Show All 63 Lines • ▼ Show 20 Lines | |||||
#include <sys/mount.h> | #include <sys/mount.h> | ||||
#include <sys/mutex.h> | #include <sys/mutex.h> | ||||
#include <sys/namei.h> | #include <sys/namei.h> | ||||
#include <sys/vnode.h> | #include <sys/vnode.h> | ||||
#include <sys/bio.h> | #include <sys/bio.h> | ||||
#include <sys/buf.h> | #include <sys/buf.h> | ||||
#include <sys/filio.h> | #include <sys/filio.h> | ||||
#include <sys/resourcevar.h> | #include <sys/resourcevar.h> | ||||
#include <sys/rmlock.h> | |||||
#include <sys/rwlock.h> | #include <sys/rwlock.h> | ||||
#include <sys/sx.h> | #include <sys/sx.h> | ||||
#include <sys/sysctl.h> | #include <sys/sysctl.h> | ||||
#include <sys/ttycom.h> | #include <sys/ttycom.h> | ||||
#include <sys/conf.h> | #include <sys/conf.h> | ||||
#include <sys/syslog.h> | #include <sys/syslog.h> | ||||
#include <sys/unistd.h> | #include <sys/unistd.h> | ||||
#include <sys/user.h> | #include <sys/user.h> | ||||
▲ Show 20 Lines • Show All 1,536 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Preparing to start a filesystem write operation. If the operation is | * Preparing to start a filesystem write operation. If the operation is | ||||
* permitted, then we bump the count of operations in progress and | * permitted, then we bump the count of operations in progress and | ||||
* proceed. If a suspend request is in progress, we wait until the | * proceed. If a suspend request is in progress, we wait until the | ||||
* suspension is over, and then proceed. | * suspension is over, and then proceed. | ||||
*/ | */ | ||||
static int | static int | ||||
vn_start_write_locked(struct mount *mp, int flags) | vn_start_write_refed(struct mount *mp, int flags, bool mplocked) | ||||
{ | { | ||||
struct rm_priotracker tracker; | |||||
int error, mflags; | int error, mflags; | ||||
if (__predict_false(mplocked)) | |||||
goto slowpath; | |||||
mtx_assert(MNT_MTX(mp), MA_NOTOWNED); | |||||
if (__predict_false(flags & V_XSLEEP)) { | |||||
MNT_ILOCK(mp); | |||||
goto slowpath; | |||||
} | |||||
if (vfs_op_thread_enter(mp, &tracker)) { | |||||
kib: I think you can write
`if (__predict_true(!mplocked && (flags & V_XSLEEP) == 0) &&… | |||||
MPASS((mp->mnt_kern_flag & MNTK_SUSPEND) == 0); | |||||
atomic_add_int(&mp->mnt_writeopcount, 1); | |||||
vfs_op_thread_exit(mp, &tracker); | |||||
return (0); | |||||
} | |||||
MNT_ILOCK(mp); | |||||
slowpath: | |||||
mtx_assert(MNT_MTX(mp), MA_OWNED); | mtx_assert(MNT_MTX(mp), MA_OWNED); | ||||
error = 0; | error = 0; | ||||
/* | /* | ||||
* Check on status of suspension. | * Check on status of suspension. | ||||
*/ | */ | ||||
if ((curthread->td_pflags & TDP_IGNSUSP) == 0 || | if ((curthread->td_pflags & TDP_IGNSUSP) == 0 || | ||||
mp->mnt_susp_owner != curthread) { | mp->mnt_susp_owner != curthread) { | ||||
mflags = ((mp->mnt_vfc->vfc_flags & VFCF_SBDRY) != 0 ? | mflags = ((mp->mnt_vfc->vfc_flags & VFCF_SBDRY) != 0 ? | ||||
(flags & PCATCH) : 0) | (PUSER - 1); | (flags & PCATCH) : 0) | (PUSER - 1); | ||||
while ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0) { | while ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0) { | ||||
if (flags & V_NOWAIT) { | if (flags & V_NOWAIT) { | ||||
error = EWOULDBLOCK; | error = EWOULDBLOCK; | ||||
goto unlock; | goto unlock; | ||||
} | } | ||||
error = msleep(&mp->mnt_flag, MNT_MTX(mp), mflags, | error = msleep(&mp->mnt_flag, MNT_MTX(mp), mflags, | ||||
"suspfs", 0); | "suspfs", 0); | ||||
if (error) | if (error) | ||||
goto unlock; | goto unlock; | ||||
} | } | ||||
} | } | ||||
if (flags & V_XSLEEP) | if (flags & V_XSLEEP) | ||||
goto unlock; | goto unlock; | ||||
mp->mnt_writeopcount++; | atomic_add_int(&mp->mnt_writeopcount, 1); | ||||
unlock: | unlock: | ||||
if (error != 0 || (flags & V_XSLEEP) != 0) | if (error != 0 || (flags & V_XSLEEP) != 0) | ||||
MNT_REL(mp); | MNT_REL(mp); | ||||
MNT_IUNLOCK(mp); | MNT_IUNLOCK(mp); | ||||
return (error); | return (error); | ||||
} | } | ||||
int | int | ||||
Show All 29 Lines | vn_start_write(struct vnode *vp, struct mount **mpp, int flags) | ||||
/* | /* | ||||
* VOP_GETWRITEMOUNT() returns with the mp refcount held through | * VOP_GETWRITEMOUNT() returns with the mp refcount held through | ||||
* a vfs_ref(). | * a vfs_ref(). | ||||
* As long as a vnode is not provided we need to acquire a | * As long as a vnode is not provided we need to acquire a | ||||
* refcount for the provided mountpoint too, in order to | * refcount for the provided mountpoint too, in order to | ||||
* emulate a vfs_ref(). | * emulate a vfs_ref(). | ||||
*/ | */ | ||||
MNT_ILOCK(mp); | |||||
if (vp == NULL && (flags & V_MNTREF) == 0) | if (vp == NULL && (flags & V_MNTREF) == 0) | ||||
MNT_REF(mp); | vfs_ref(mp); | ||||
return (vn_start_write_locked(mp, flags)); | return (vn_start_write_refed(mp, flags, false)); | ||||
} | } | ||||
/* | /* | ||||
* Secondary suspension. Used by operations such as vop_inactive | * Secondary suspension. Used by operations such as vop_inactive | ||||
* routines that are needed by the higher level functions. These | * routines that are needed by the higher level functions. These | ||||
* are allowed to proceed until all the higher level functions have | * are allowed to proceed until all the higher level functions have | ||||
* completed (indicated by mnt_writeopcount dropping to zero). At that | * completed (indicated by mnt_writeopcount dropping to zero). At that | ||||
* time, these operations are halted until the suspension is over. | * time, these operations are halted until the suspension is over. | ||||
▲ Show 20 Lines • Show All 65 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Filesystem write operation has completed. If we are suspending and this | * Filesystem write operation has completed. If we are suspending and this | ||||
* operation is the last one, notify the suspender that the suspension is | * operation is the last one, notify the suspender that the suspension is | ||||
* now in effect. | * now in effect. | ||||
*/ | */ | ||||
void | void | ||||
vn_finished_write(struct mount *mp) | vn_finished_write(struct mount *mp) | ||||
{ | { | ||||
struct rm_priotracker tracker; | |||||
int c; | |||||
if (mp == NULL || !vn_suspendable(mp)) | if (mp == NULL || !vn_suspendable(mp)) | ||||
return; | return; | ||||
if (vfs_op_thread_enter(mp, &tracker)) { | |||||
c = atomic_fetchadd_int(&mp->mnt_writeopcount, -1) - 1; | |||||
if (c < 0) | |||||
panic("vn_finished_write: invalid writeopcount %d", c); | |||||
Not Done Inline ActionsSince we do this twice it could just be part of a macro or inline. jeff: Since we do this twice it could just be part of a macro or inline. | |||||
MNT_REL_FASTPATH(mp); | |||||
vfs_op_thread_exit(mp, &tracker); | |||||
return; | |||||
} | |||||
MNT_ILOCK(mp); | MNT_ILOCK(mp); | ||||
MNT_REL(mp); | MNT_REL(mp); | ||||
mp->mnt_writeopcount--; | c = atomic_fetchadd_int(&mp->mnt_writeopcount, -1) - 1; | ||||
if (mp->mnt_writeopcount < 0) | if (c < 0) | ||||
panic("vn_finished_write: neg cnt"); | panic("vn_finished_write: invalid writeopcount %d", c); | ||||
if ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0 && | if ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0 && c == 0) | ||||
mp->mnt_writeopcount <= 0) | |||||
wakeup(&mp->mnt_writeopcount); | wakeup(&mp->mnt_writeopcount); | ||||
MNT_IUNLOCK(mp); | MNT_IUNLOCK(mp); | ||||
} | } | ||||
/* | /* | ||||
* Filesystem secondary write operation has completed. If we are | * Filesystem secondary write operation has completed. If we are | ||||
* suspending and this operation is the last one, notify the suspender | * suspending and this operation is the last one, notify the suspender | ||||
Show All 22 Lines | |||||
*/ | */ | ||||
int | int | ||||
vfs_write_suspend(struct mount *mp, int flags) | vfs_write_suspend(struct mount *mp, int flags) | ||||
{ | { | ||||
int error; | int error; | ||||
MPASS(vn_suspendable(mp)); | MPASS(vn_suspendable(mp)); | ||||
vfs_op_enter(mp); | |||||
MNT_ILOCK(mp); | MNT_ILOCK(mp); | ||||
if (mp->mnt_susp_owner == curthread) { | if (mp->mnt_susp_owner == curthread) { | ||||
vfs_op_exit_locked(mp); | |||||
MNT_IUNLOCK(mp); | MNT_IUNLOCK(mp); | ||||
return (EALREADY); | return (EALREADY); | ||||
} | } | ||||
while (mp->mnt_kern_flag & MNTK_SUSPEND) | while (mp->mnt_kern_flag & MNTK_SUSPEND) | ||||
msleep(&mp->mnt_flag, MNT_MTX(mp), PUSER - 1, "wsuspfs", 0); | msleep(&mp->mnt_flag, MNT_MTX(mp), PUSER - 1, "wsuspfs", 0); | ||||
/* | /* | ||||
* Unmount holds a write reference on the mount point. If we | * Unmount holds a write reference on the mount point. If we | ||||
* own busy reference and drain for writers, we deadlock with | * own busy reference and drain for writers, we deadlock with | ||||
* the reference draining in the unmount path. Callers of | * the reference draining in the unmount path. Callers of | ||||
* vfs_write_suspend() must specify VS_SKIP_UNMOUNT if | * vfs_write_suspend() must specify VS_SKIP_UNMOUNT if | ||||
* vfs_busy() reference is owned and caller is not in the | * vfs_busy() reference is owned and caller is not in the | ||||
* unmount context. | * unmount context. | ||||
*/ | */ | ||||
if ((flags & VS_SKIP_UNMOUNT) != 0 && | if ((flags & VS_SKIP_UNMOUNT) != 0 && | ||||
(mp->mnt_kern_flag & MNTK_UNMOUNT) != 0) { | (mp->mnt_kern_flag & MNTK_UNMOUNT) != 0) { | ||||
vfs_op_exit_locked(mp); | |||||
MNT_IUNLOCK(mp); | MNT_IUNLOCK(mp); | ||||
return (EBUSY); | return (EBUSY); | ||||
} | } | ||||
mp->mnt_kern_flag |= MNTK_SUSPEND; | mp->mnt_kern_flag |= MNTK_SUSPEND; | ||||
mp->mnt_susp_owner = curthread; | mp->mnt_susp_owner = curthread; | ||||
if (mp->mnt_writeopcount > 0) | if (mp->mnt_writeopcount > 0) | ||||
(void) msleep(&mp->mnt_writeopcount, | (void) msleep(&mp->mnt_writeopcount, | ||||
MNT_MTX(mp), (PUSER - 1)|PDROP, "suspwt", 0); | MNT_MTX(mp), (PUSER - 1)|PDROP, "suspwt", 0); | ||||
else | else | ||||
MNT_IUNLOCK(mp); | MNT_IUNLOCK(mp); | ||||
if ((error = VFS_SYNC(mp, MNT_SUSPEND)) != 0) | if ((error = VFS_SYNC(mp, MNT_SUSPEND)) != 0) { | ||||
vfs_write_resume(mp, 0); | vfs_write_resume(mp, 0); | ||||
vfs_op_exit(mp); | |||||
} | |||||
return (error); | return (error); | ||||
} | } | ||||
/* | /* | ||||
* Request a filesystem to resume write operations. | * Request a filesystem to resume write operations. | ||||
*/ | */ | ||||
void | void | ||||
vfs_write_resume(struct mount *mp, int flags) | vfs_write_resume(struct mount *mp, int flags) | ||||
{ | { | ||||
MPASS(vn_suspendable(mp)); | MPASS(vn_suspendable(mp)); | ||||
MNT_ILOCK(mp); | MNT_ILOCK(mp); | ||||
if ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0) { | if ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0) { | ||||
KASSERT(mp->mnt_susp_owner == curthread, ("mnt_susp_owner")); | KASSERT(mp->mnt_susp_owner == curthread, ("mnt_susp_owner")); | ||||
mp->mnt_kern_flag &= ~(MNTK_SUSPEND | MNTK_SUSPEND2 | | mp->mnt_kern_flag &= ~(MNTK_SUSPEND | MNTK_SUSPEND2 | | ||||
MNTK_SUSPENDED); | MNTK_SUSPENDED); | ||||
mp->mnt_susp_owner = NULL; | mp->mnt_susp_owner = NULL; | ||||
wakeup(&mp->mnt_writeopcount); | wakeup(&mp->mnt_writeopcount); | ||||
wakeup(&mp->mnt_flag); | wakeup(&mp->mnt_flag); | ||||
curthread->td_pflags &= ~TDP_IGNSUSP; | curthread->td_pflags &= ~TDP_IGNSUSP; | ||||
if ((flags & VR_START_WRITE) != 0) { | if ((flags & VR_START_WRITE) != 0) { | ||||
MNT_REF(mp); | MNT_REF(mp); | ||||
mp->mnt_writeopcount++; | atomic_add_int(&mp->mnt_writeopcount, 1); | ||||
} | } | ||||
MNT_IUNLOCK(mp); | MNT_IUNLOCK(mp); | ||||
if ((flags & VR_NO_SUSPCLR) == 0) | if ((flags & VR_NO_SUSPCLR) == 0) | ||||
VFS_SUSP_CLEAN(mp); | VFS_SUSP_CLEAN(mp); | ||||
vfs_op_exit(mp); | |||||
} else if ((flags & VR_START_WRITE) != 0) { | } else if ((flags & VR_START_WRITE) != 0) { | ||||
MNT_REF(mp); | MNT_REF(mp); | ||||
vn_start_write_locked(mp, 0); | vn_start_write_refed(mp, 0, true); | ||||
} else { | } else { | ||||
MNT_IUNLOCK(mp); | MNT_IUNLOCK(mp); | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Helper loop around vfs_write_suspend() for filesystem unmount VFS | * Helper loop around vfs_write_suspend() for filesystem unmount VFS | ||||
* methods. | * methods. | ||||
▲ Show 20 Lines • Show All 1,195 Lines • Show Last 20 Lines |
I think you can write
if (__predict_true(!mplocked && (flags & V_XSLEEP) == 0) && vfs_op_thread_enter(mp)) {
and remove that ugly slowpath label.