Page MenuHomeFreeBSD
Files F163579128

D21425.id61323.diff
No OneTemporary
Actions

D21425.id61323.diff
View Options

Index: sys/cddl/compat/opensolaris/kern/opensolaris_vfs.c
===================================================================
--- sys/cddl/compat/opensolaris/kern/opensolaris_vfs.c
+++ sys/cddl/compat/opensolaris/kern/opensolaris_vfs.c
@@ -242,6 +242,7 @@
if (VFS_ROOT(mp, LK_EXCLUSIVE, &mvp))
panic("mount: lost mount");
VOP_UNLOCK(vp, 0);
+ vfs_fastpath_enable_mp(mp);
vfs_unbusy(mp);
*vpp = mvp;
return (0);
Index: sys/kern/vfs_default.c
===================================================================
--- sys/kern/vfs_default.c
+++ sys/kern/vfs_default.c
@@ -51,6 +51,7 @@
#include <sys/malloc.h>
#include <sys/mount.h>
#include <sys/namei.h>
+#include <sys/rmlock.h>
#include <sys/rwlock.h>
#include <sys/fcntl.h>
#include <sys/unistd.h>
@@ -586,6 +587,7 @@
struct mount **a_mpp;
} */ *ap;
{
+ struct rm_priotracker tracker;
struct mount *mp;
struct vnode *vp;
@@ -598,6 +600,19 @@
*/
vp = ap->a_vp;
mp = vp->v_mount;
+ if (mp == NULL)
+ goto out;
+ if (vfs_fastpath_enter_mp(mp, &tracker)) {
+ if (mp != vp->v_mount) {
+ vfs_fastpath_exit_mp(mp, &tracker);
+ mp = NULL;
+ goto out;
+ }
+ MNT_REF_FASTPATH(mp);
+ vfs_fastpath_exit_mp(mp, &tracker);
+ goto out;
+ }
+
MNT_ILOCK(mp);
if (mp != vp->v_mount) {
MNT_IUNLOCK(mp);
Index: sys/kern/vfs_mount.c
===================================================================
--- sys/kern/vfs_mount.c
+++ sys/kern/vfs_mount.c
@@ -58,6 +58,7 @@
#include <sys/sbuf.h>
#include <sys/syscallsubr.h>
#include <sys/sysproto.h>
+#include <sys/rmlock.h>
#include <sys/sx.h>
#include <sys/sysctl.h>
#include <sys/sysent.h>
@@ -77,6 +78,7 @@
static int vfs_domount(struct thread *td, const char *fstype, char *fspath,
uint64_t fsflags, struct vfsoptlist **optlist);
static void free_mntarg(struct mntarg *ma);
+static void vfs_fastpath_disable_mp_new(struct mount *mp);
static int usermount = 0;
SYSCTL_INT(_vfs, OID_AUTO, usermount, CTLFLAG_RW, &usermount, 0,
@@ -123,6 +125,9 @@
mtx_init(&mp->mnt_mtx, "struct mount mtx", NULL, MTX_DEF);
mtx_init(&mp->mnt_listmtx, "struct mount vlist mtx", NULL, MTX_DEF);
lockinit(&mp->mnt_explock, PVFS, "explock", 0, 0);
+ rm_init(&mp->mnt_rmlock, "struct mount rmlock");
+ mp->mnt_ref = 0;
+ mp->mnt_fastpathdisabled = 0;
return (0);
}
@@ -132,6 +137,7 @@
struct mount *mp;
mp = (struct mount *)mem;
+ rm_destroy(&mp->mnt_rmlock);
lockdestroy(&mp->mnt_explock);
mtx_destroy(&mp->mnt_listmtx);
mtx_destroy(&mp->mnt_mtx);
@@ -443,8 +449,15 @@
void
vfs_ref(struct mount *mp)
{
+ struct rm_priotracker tracker;
CTR2(KTR_VFS, "%s: mp %p", __func__, mp);
+ if (vfs_fastpath_enter_mp(mp, &tracker)) {
+ MNT_REF_FASTPATH(mp);
+ vfs_fastpath_exit_mp(mp, &tracker);
+ return;
+ }
+
MNT_ILOCK(mp);
MNT_REF(mp);
MNT_IUNLOCK(mp);
@@ -453,8 +466,15 @@
void
vfs_rel(struct mount *mp)
{
+ struct rm_priotracker tracker;
CTR2(KTR_VFS, "%s: mp %p", __func__, mp);
+ if (vfs_fastpath_enter_mp(mp, &tracker)) {
+ MNT_REL_FASTPATH(mp);
+ vfs_fastpath_exit_mp(mp, &tracker);
+ return;
+ }
+
MNT_ILOCK(mp);
MNT_REL(mp);
MNT_IUNLOCK(mp);
@@ -478,7 +498,10 @@
mp->mnt_activevnodelistsize = 0;
TAILQ_INIT(&mp->mnt_tmpfreevnodelist);
mp->mnt_tmpfreevnodelistsize = 0;
- mp->mnt_ref = 0;
+ if (mp->mnt_ref != 0 || mp->mnt_lockref != 0 ||
+ mp->mnt_writeopcount != 0)
+ panic("non-zero counters on new mp %p\n", mp);
+ vfs_fastpath_disable_mp_new(mp);
(void) vfs_busy(mp, MBF_NOWAIT);
atomic_add_acq_int(&vfsp->vfc_refcount, 1);
mp->mnt_op = vfsp->vfc_vfsops;
@@ -507,6 +530,9 @@
vfs_mount_destroy(struct mount *mp)
{
+ if (mp->mnt_fastpathdisabled == 0)
+ panic("%s: entered with fast path enabled\n", __func__);
+
MNT_ILOCK(mp);
mp->mnt_kern_flag |= MNTK_REFEXPIRE;
if (mp->mnt_kern_flag & MNTK_MWAIT) {
@@ -540,6 +566,13 @@
if (mp->mnt_lockref != 0)
panic("vfs_mount_destroy: nonzero lock refcount");
MNT_IUNLOCK(mp);
+
+ vfs_fastpath_enable_mp(mp);
+
+ if (mp->mnt_fastpathdisabled != 0)
+ panic("%s: wrong fastpathdisabled count %d\n", __func__,
+ mp->mnt_fastpathdisabled);
+
if (mp->mnt_vnodecovered != NULL)
vrele(mp->mnt_vnodecovered);
#ifdef MAC
@@ -951,6 +984,7 @@
vrele(newdp);
if ((mp->mnt_flag & MNT_RDONLY) == 0)
vfs_allocate_syncvnode(mp);
+ vfs_fastpath_enable_mp(mp);
vfs_unbusy(mp);
return (0);
}
@@ -1019,6 +1053,8 @@
VI_UNLOCK(vp);
VOP_UNLOCK(vp, 0);
+ vfs_fastpath_disable_mp(mp);
+
MNT_ILOCK(mp);
if ((mp->mnt_kern_flag & MNTK_UNMOUNT) != 0) {
MNT_IUNLOCK(mp);
@@ -1100,6 +1136,7 @@
else
vfs_deallocate_syncvnode(mp);
end:
+ vfs_fastpath_enable_mp(mp);
vfs_unbusy(mp);
VI_LOCK(vp);
vp->v_iflag &= ~VI_MOUNT;
@@ -1336,6 +1373,43 @@
vn_finished_write(mp);
}
+void
+vfs_fastpath_disable_mp(struct mount *mp)
+{
+
+ rm_wlock(&mp->mnt_rmlock);
+ MNT_ILOCK(mp);
+ mp->mnt_fastpathdisabled++;
+ if (mp->mnt_fastpathdisabled > 1) {
+ MNT_IUNLOCK(mp);
+ rm_wunlock(&mp->mnt_rmlock);
+ return;
+ }
+ MNT_IUNLOCK(mp);
+ rm_wunlock(&mp->mnt_rmlock);
+}
+
+void
+vfs_fastpath_enable_mp(struct mount *mp)
+{
+
+ MNT_ILOCK(mp);
+ if (mp->mnt_fastpathdisabled <= 0)
+ panic("invalid slowpath count %d for mp %p\n",
+ mp->mnt_fastpathdisabled, mp);
+ mp->mnt_fastpathdisabled--;
+ MNT_IUNLOCK(mp);
+}
+
+static void
+vfs_fastpath_disable_mp_new(struct mount *mp)
+{
+
+ if (mp->mnt_fastpathdisabled > 0)
+ panic("fast path already disabled for %p\n", mp);
+ mp->mnt_fastpathdisabled = 1;
+}
+
/*
* Do the actual filesystem unmount.
*/
@@ -1379,13 +1453,16 @@
return (error);
}
+ vfs_fastpath_disable_mp(mp);
+
vn_start_write(NULL, &mp, V_WAIT | V_MNTREF);
MNT_ILOCK(mp);
if ((mp->mnt_kern_flag & MNTK_UNMOUNT) != 0 ||
(mp->mnt_flag & MNT_UPDATE) != 0 ||
!TAILQ_EMPTY(&mp->mnt_uppers)) {
dounmount_cleanup(mp, coveredvp, 0);
- return (EBUSY);
+ error = EBUSY;
+ goto out_error;
}
mp->mnt_kern_flag |= MNTK_UNMOUNT;
if (flags & MNT_NONBUSY) {
@@ -1394,7 +1471,7 @@
MNT_ILOCK(mp);
if (error != 0) {
dounmount_cleanup(mp, coveredvp, MNTK_UNMOUNT);
- return (error);
+ goto out_error;
}
}
/* Allow filesystems to detect that a forced unmount is in progress. */
@@ -1472,7 +1549,7 @@
MNT_IUNLOCK(mp);
if (coveredvp)
VOP_UNLOCK(coveredvp, 0);
- return (error);
+ goto out_error;
}
mtx_lock(&mountlist_mtx);
TAILQ_REMOVE(&mountlist, mp, mnt_list);
@@ -1491,6 +1568,9 @@
rootdevmp = NULL;
vfs_mount_destroy(mp);
return (0);
+out_error:
+ vfs_fastpath_enable_mp(mp);
+ return (error);
}
/*
Index: sys/kern/vfs_mountroot.c
===================================================================
--- sys/kern/vfs_mountroot.c
+++ sys/kern/vfs_mountroot.c
@@ -273,6 +273,7 @@
*mpp = mp;
rootdevmp = mp;
+ vfs_fastpath_enable_mp(mp);
}
set_rootvnode();
Index: sys/sys/mount.h
===================================================================
--- sys/sys/mount.h
+++ sys/sys/mount.h
@@ -41,8 +41,10 @@
#include <sys/lock.h>
#include <sys/lockmgr.h>
#include <sys/tslog.h>
+#include <sys/_cpuset.h>
#include <sys/_mutex.h>
#include <sys/_sx.h>
+#include <sys/_rmlock.h>
#endif
/*
@@ -226,6 +228,8 @@
struct lock mnt_explock; /* vfs_export walkers lock */
TAILQ_ENTRY(mount) mnt_upper_link; /* (m) we in the all uppers */
TAILQ_HEAD(, mount) mnt_uppers; /* (m) upper mounts over us*/
+ struct rmlock mnt_rmlock;
+ int mnt_fastpathdisabled; /* (i) is fast path disabled */
};
/*
@@ -265,15 +269,26 @@
#define MNT_ITRYLOCK(mp) mtx_trylock(&(mp)->mnt_mtx)
#define MNT_IUNLOCK(mp) mtx_unlock(&(mp)->mnt_mtx)
#define MNT_MTX(mp) (&(mp)->mnt_mtx)
+
+#define MNT_REF_FASTPATH(mp) do { \
+ atomic_add_int(&(mp)->mnt_ref, 1); \
+} while (0)
+#define MNT_REL_FASTPATH(mp) do { \
+ int _c; \
+ _c = atomic_fetchadd_int(&(mp)->mnt_ref, -1) - 1; \
+ KASSERT(_c >= 0, ("negative mnt_ref %d", _c)); \
+} while (0)
+
#define MNT_REF(mp) do { \
mtx_assert(MNT_MTX(mp), MA_OWNED); \
- (mp)->mnt_ref++; \
+ atomic_add_int(&(mp)->mnt_ref, 1); \
} while (0)
#define MNT_REL(mp) do { \
+ int _c; \
mtx_assert(MNT_MTX(mp), MA_OWNED); \
- KASSERT((mp)->mnt_ref > 0, ("negative mnt_ref")); \
- (mp)->mnt_ref--; \
- if ((mp)->mnt_ref == 0) \
+ _c = atomic_fetchadd_int(&(mp)->mnt_ref, -1) - 1; \
+ KASSERT(_c >= 0, ("negative mnt_ref %d", _c)); \
+ if (_c == 0) \
wakeup((mp)); \
} while (0)
@@ -929,6 +944,24 @@
void syncer_suspend(void);
void syncer_resume(void);
+void vfs_fastpath_disable_mp(struct mount *);
+void vfs_fastpath_enable_mp(struct mount *);
+
+#define vfs_fastpath_enter_mp(mp, tracker) ({ \
+ struct mount *_mp = (mp); \
+ int _retval; \
+ _retval = rm_try_rlock(&(_mp)->mnt_rmlock, tracker); \
+ if (__predict_true(_retval != 0)) { \
+ if (__predict_false(_mp->mnt_fastpathdisabled)) {\
+ vfs_fastpath_exit_mp(_mp, tracker); \
+ _retval = 0; \
+ } \
+ } \
+ _retval; \
+})
+
+#define vfs_fastpath_exit_mp(mp, tracker) rm_runlock(&(mp)->mnt_rmlock, tracker)
+
#else /* !_KERNEL */
#include <sys/cdefs.h>

File Metadata

Mime Type
text/plain
Expires
Sat, Jul 25, 4:32 PM (4 h, 24 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
35479775
Default Alt Text
D21425.id61323.diff (8 KB)

Event Timeline