Index: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_znode.h =================================================================== --- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_znode.h +++ head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_znode.h @@ -246,6 +246,8 @@ #define VTOZ(VP) ((znode_t *)(VP)->v_data) #endif +#define VTOZ_SMR(VP) ((znode_t *)vn_load_v_data_smr(VP)) + /* Called on entry to each ZFS vnode and vfs operation */ #define ZFS_ENTER(zfsvfs) \ { \ Index: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_acl.c =================================================================== --- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_acl.c +++ head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_acl.c @@ -1146,6 +1146,7 @@ ASSERT_VOP_ELOCKED(ZTOV(zp), __func__); ASSERT(MUTEX_HELD(&zp->z_acl_lock)); + ASSERT_VOP_IN_SEQC(ZTOV(zp)); if ((error = zfs_acl_node_read(zp, &aclp, B_FALSE)) == 0) zp->z_mode = zfs_mode_compute(zp->z_mode, aclp, @@ -1172,6 +1173,8 @@ uint64_t ctime[2]; int count = 0; zfs_acl_phys_t acl_phys; + + ASSERT_VOP_IN_SEQC(ZTOV(zp)); mode = zp->z_mode; Index: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c =================================================================== --- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c +++ head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c @@ -1391,6 +1391,9 @@ vfsp->vfs_data = zfsvfs; vfsp->mnt_flag |= MNT_LOCAL; +#if defined(_KERNEL) && !defined(KMEM_DEBUG) + vfsp->mnt_kern_flag |= MNTK_FPLOOKUP; +#endif vfsp->mnt_kern_flag |= MNTK_LOOKUP_SHARED; vfsp->mnt_kern_flag |= MNTK_SHARED_WRITES; vfsp->mnt_kern_flag |= MNTK_EXTENDED_SHARED; Index: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c =================================================================== --- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c +++ head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -78,6 +79,8 @@ #include #include +VFS_SMR_DECLARE; + /* * Programming rules. * @@ -3698,6 +3701,7 @@ char *snm = scnp->cn_nameptr; char *tnm = tcnp->cn_nameptr; int error = 0; + bool want_seqc_end = false; /* Reject renames across filesystems. */ if ((*svpp)->v_mount != tdvp->v_mount || @@ -3828,6 +3832,14 @@ } } + vn_seqc_write_begin(*svpp); + vn_seqc_write_begin(sdvp); + if (*tvpp != NULL) + vn_seqc_write_begin(*tvpp); + if (tdvp != *tvpp) + vn_seqc_write_begin(tdvp); + want_seqc_end = true; + vnevent_rename_src(*svpp, sdvp, scnp->cn_nameptr, ct); if (tzp) vnevent_rename_dest(*tvpp, tdvp, tnm, ct); @@ -3914,10 +3926,20 @@ unlockout: /* all 4 vnodes are locked, ZFS_ENTER called */ ZFS_EXIT(zfsvfs); + if (want_seqc_end) { + vn_seqc_write_end(*svpp); + vn_seqc_write_end(sdvp); + if (*tvpp != NULL) + vn_seqc_write_end(*tvpp); + if (tdvp != *tvpp) + vn_seqc_write_end(tdvp); + want_seqc_end = false; + } VOP_UNLOCK(*svpp); VOP_UNLOCK(sdvp); out: /* original two vnodes are locked */ + MPASS(!want_seqc_end); if (error == 0 && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS) zil_commit(zilog, 0); @@ -4861,7 +4883,32 @@ ap->a_cred, NULL)); } +/* + * VOP_FPLOOKUP_VEXEC routines are subject to special circumstances, see + * the comment above cache_fplookup for details. + */ static int +zfs_freebsd_fplookup_vexec(struct vop_fplookup_vexec_args *v) +{ + vnode_t *vp; + znode_t *zp; + uint64_t pflags; + + vp = v->a_vp; + zp = VTOZ_SMR(vp); + if (__predict_false(zp == NULL)) + return (EAGAIN); + pflags = atomic_load_64(&zp->z_pflags); + if (pflags & ZFS_AV_QUARANTINED) + return (EAGAIN); + if (pflags & ZFS_XATTR) + return (EAGAIN); + if ((pflags & ZFS_NO_EXECS_DENIED) == 0) + return (EAGAIN); + return (0); +} + +static int zfs_freebsd_access(ap) struct vop_access_args /* { struct vnode *a_vp; @@ -5998,6 +6045,7 @@ .vop_inactive = zfs_freebsd_inactive, .vop_need_inactive = zfs_freebsd_need_inactive, .vop_reclaim = zfs_freebsd_reclaim, + .vop_fplookup_vexec = zfs_freebsd_fplookup_vexec, .vop_access = zfs_freebsd_access, .vop_allocate = VOP_EINVAL, .vop_lookup = zfs_cache_lookup, @@ -6066,6 +6114,7 @@ */ struct vop_vector zfs_shareops = { .vop_default = &default_vnodeops, + .vop_fplookup_vexec = zfs_freebsd_fplookup_vexec, .vop_access = zfs_freebsd_access, .vop_inactive = zfs_freebsd_inactive, .vop_reclaim = zfs_freebsd_reclaim, Index: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c =================================================================== --- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c +++ head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c @@ -99,7 +99,12 @@ */ krwlock_t zfsvfs_lock; +#if defined(_KERNEL) && !defined(KMEM_DEBUG) +#define _ZFS_USE_SMR +static uma_zone_t znode_uma_zone; +#else static kmem_cache_t *znode_cache = NULL; +#endif /*ARGSUSED*/ static void @@ -366,6 +371,23 @@ } #endif /* illumos */ +#ifdef _ZFS_USE_SMR +VFS_SMR_DECLARE; + +static int +zfs_znode_cache_constructor_smr(void *mem, int size __unused, void *private, int flags) +{ + + return (zfs_znode_cache_constructor(mem, private, flags)); +} + +static void +zfs_znode_cache_destructor_smr(void *mem, int size __unused, void *private) +{ + + zfs_znode_cache_destructor(mem, private); +} + void zfs_znode_init(void) { @@ -373,6 +395,34 @@ * Initialize zcache */ rw_init(&zfsvfs_lock, NULL, RW_DEFAULT, NULL); + ASSERT(znode_uma_zone == NULL); + znode_uma_zone = uma_zcreate("zfs_znode_cache", + sizeof (znode_t), zfs_znode_cache_constructor_smr, + zfs_znode_cache_destructor_smr, NULL, NULL, 0, 0); + VFS_SMR_ZONE_SET(znode_uma_zone); +} + +static znode_t * +zfs_znode_alloc_kmem(int flags) +{ + + return (uma_zalloc_smr(znode_uma_zone, flags)); +} + +static void +zfs_znode_free_kmem(znode_t *zp) +{ + + uma_zfree_smr(znode_uma_zone, zp); +} +#else +void +zfs_znode_init(void) +{ + /* + * Initialize zcache + */ + rw_init(&zfsvfs_lock, NULL, RW_DEFAULT, NULL); ASSERT(znode_cache == NULL); znode_cache = kmem_cache_create("zfs_znode_cache", sizeof (znode_t), 0, zfs_znode_cache_constructor, @@ -380,6 +430,21 @@ kmem_cache_set_move(znode_cache, zfs_znode_move); } +static znode_t * +zfs_znode_alloc_kmem(int flags) +{ + + return (kmem_cache_alloc(znode_cache, flags)); +} + +static void +zfs_znode_free_kmem(znode_t *zp) +{ + + kmem_cache_free(znode_cache, zp); +} +#endif + void zfs_znode_fini(void) { @@ -393,9 +458,17 @@ /* * Cleanup zcache */ - if (znode_cache) +#ifdef _ZFS_USE_SMR + if (znode_uma_zone) { + uma_zdestroy(znode_uma_zone); + znode_uma_zone = NULL; + } +#else + if (znode_cache) { kmem_cache_destroy(znode_cache); - znode_cache = NULL; + znode_cache = NULL; + } +#endif rw_destroy(&zfsvfs_lock); } @@ -508,7 +581,7 @@ vattr.va_uid = crgetuid(kcred); vattr.va_gid = crgetgid(kcred); - sharezp = kmem_cache_alloc(znode_cache, KM_SLEEP); + sharezp = zfs_znode_alloc_kmem(KM_SLEEP); ASSERT(!POINTER_IS_VALID(sharezp->z_zfsvfs)); sharezp->z_moved = 0; sharezp->z_unlinked = 0; @@ -527,7 +600,7 @@ zfs_acl_ids_free(&acl_ids); sa_handle_destroy(sharezp->z_sa_hdl); - kmem_cache_free(znode_cache, sharezp); + zfs_znode_free_kmem(sharezp); return (error); } @@ -642,13 +715,18 @@ int count = 0; int error; - zp = kmem_cache_alloc(znode_cache, KM_SLEEP); + zp = zfs_znode_alloc_kmem(KM_SLEEP); +#ifndef _ZFS_USE_SMR + KASSERT((zfsvfs->z_parent->z_vfs->mnt_kern_flag & MNTK_FPLOOKUP) == 0, + ("%s: fast path lookup enabled without smr", __func__)); +#endif + KASSERT(curthread->td_vp_reserved != NULL, ("zfs_znode_alloc: getnewvnode without preallocated vnode")); error = getnewvnode("zfs", zfsvfs->z_parent->z_vfs, &zfs_vnodeops, &vp); if (error != 0) { - kmem_cache_free(znode_cache, zp); + zfs_znode_free_kmem(zp); return (NULL); } zp->z_vnode = vp; @@ -695,7 +773,7 @@ sa_handle_destroy(zp->z_sa_hdl); zfs_vnode_forget(vp); zp->z_vnode = NULL; - kmem_cache_free(znode_cache, zp); + zfs_znode_free_kmem(zp); return (NULL); } @@ -1061,6 +1139,8 @@ xoap = xva_getxoptattr(xvap); ASSERT(xoap); + ASSERT_VOP_IN_SEQC(ZTOV(zp)); + if (XVA_ISSET_REQ(xvap, XAT_CREATETIME)) { uint64_t times[2]; ZFS_TIME_ENCODE(&xoap->xoa_createtime, times); @@ -1490,7 +1570,7 @@ zp->z_acl_cached = NULL; } - kmem_cache_free(znode_cache, zp); + zfs_znode_free_kmem(zp); #ifdef illumos VFS_RELE(zfsvfs->z_vfs); @@ -1950,7 +2030,7 @@ zfsvfs = kmem_zalloc(sizeof (zfsvfs_t), KM_SLEEP); - rootzp = kmem_cache_alloc(znode_cache, KM_SLEEP); + rootzp = zfs_znode_alloc_kmem(KM_SLEEP); ASSERT(!POINTER_IS_VALID(rootzp->z_zfsvfs)); rootzp->z_moved = 0; rootzp->z_unlinked = 0; @@ -1994,7 +2074,7 @@ POINTER_INVALIDATE(&rootzp->z_zfsvfs); sa_handle_destroy(rootzp->z_sa_hdl); - kmem_cache_free(znode_cache, rootzp); + zfs_znode_free_kmem(rootzp); /* * Create shares directory