Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/vfs_cache.c
Show First 20 Lines • Show All 556 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* UMA zones. | * UMA zones. | ||||
*/ | */ | ||||
static uma_zone_t __read_mostly cache_zone_small; | static uma_zone_t __read_mostly cache_zone_small; | ||||
static uma_zone_t __read_mostly cache_zone_small_ts; | static uma_zone_t __read_mostly cache_zone_small_ts; | ||||
static uma_zone_t __read_mostly cache_zone_large; | static uma_zone_t __read_mostly cache_zone_large; | ||||
static uma_zone_t __read_mostly cache_zone_large_ts; | static uma_zone_t __read_mostly cache_zone_large_ts; | ||||
char * | |||||
cache_symlink_alloc(size_t size, int flags) | |||||
{ | |||||
if (size < CACHE_ZONE_SMALL_SIZE) { | |||||
return (uma_zalloc_smr(cache_zone_small, flags)); | |||||
} | |||||
if (size < CACHE_ZONE_LARGE_SIZE) { | |||||
return (uma_zalloc_smr(cache_zone_large, flags)); | |||||
} | |||||
return (NULL); | |||||
} | |||||
void | |||||
cache_symlink_free(char *string, size_t size) | |||||
{ | |||||
MPASS(string != NULL); | |||||
if (size < CACHE_ZONE_SMALL_SIZE) { | |||||
uma_zfree_smr(cache_zone_small, string); | |||||
return; | |||||
} | |||||
if (size < CACHE_ZONE_LARGE_SIZE) { | |||||
uma_zfree_smr(cache_zone_large, string); | |||||
return; | |||||
} | |||||
__assert_unreachable(); | |||||
} | |||||
static struct namecache * | static struct namecache * | ||||
cache_alloc_uma(int len, bool ts) | cache_alloc_uma(int len, bool ts) | ||||
{ | { | ||||
struct namecache_ts *ncp_ts; | struct namecache_ts *ncp_ts; | ||||
struct namecache *ncp; | struct namecache *ncp; | ||||
if (__predict_false(ts)) { | if (__predict_false(ts)) { | ||||
if (len <= CACHE_PATH_CUTOFF) | if (len <= CACHE_PATH_CUTOFF) | ||||
▲ Show 20 Lines • Show All 3,006 Lines • ▼ Show 20 Lines | |||||
void | void | ||||
cache_fast_lookup_enabled_recalc(void) | cache_fast_lookup_enabled_recalc(void) | ||||
{ | { | ||||
int lookup_flag; | int lookup_flag; | ||||
int mac_on; | int mac_on; | ||||
#ifdef MAC | #ifdef MAC | ||||
mac_on = mac_vnode_check_lookup_enabled(); | mac_on = mac_vnode_check_lookup_enabled(); | ||||
mac_on |= mac_vnode_check_readlink_enabled(); | |||||
#else | #else | ||||
mac_on = 0; | mac_on = 0; | ||||
#endif | #endif | ||||
lookup_flag = atomic_load_int(&cache_fast_lookup); | lookup_flag = atomic_load_int(&cache_fast_lookup); | ||||
if (lookup_flag && !mac_on) { | if (lookup_flag && !mac_on) { | ||||
atomic_store_char(&cache_fast_lookup_enabled, true); | atomic_store_char(&cache_fast_lookup_enabled, true); | ||||
} else { | } else { | ||||
Show All 15 Lines | |||||
SYSCTL_PROC(_vfs, OID_AUTO, cache_fast_lookup, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_MPSAFE, | SYSCTL_PROC(_vfs, OID_AUTO, cache_fast_lookup, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_MPSAFE, | ||||
&cache_fast_lookup, 0, syscal_vfs_cache_fast_lookup, "IU", ""); | &cache_fast_lookup, 0, syscal_vfs_cache_fast_lookup, "IU", ""); | ||||
/* | /* | ||||
* Components of nameidata (or objects it can point to) which may | * Components of nameidata (or objects it can point to) which may | ||||
* need restoring in case fast path lookup fails. | * need restoring in case fast path lookup fails. | ||||
*/ | */ | ||||
struct nameidata_outer { | struct nameidata_outer { | ||||
size_t ni_pathlen; | |||||
int cn_flags; | int cn_flags; | ||||
}; | }; | ||||
struct nameidata_saved { | struct nameidata_saved { | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
char *cn_nameptr; | char *cn_nameptr; | ||||
size_t ni_pathlen; | size_t ni_pathlen; | ||||
#endif | #endif | ||||
Show All 25 Lines | #ifdef INVARIANTS | ||||
struct cache_fpl_debug debug; | struct cache_fpl_debug debug; | ||||
#endif | #endif | ||||
}; | }; | ||||
static bool cache_fplookup_is_mp(struct cache_fpl *fpl); | static bool cache_fplookup_is_mp(struct cache_fpl *fpl); | ||||
static int cache_fplookup_cross_mount(struct cache_fpl *fpl); | static int cache_fplookup_cross_mount(struct cache_fpl *fpl); | ||||
static int cache_fplookup_partial_setup(struct cache_fpl *fpl); | static int cache_fplookup_partial_setup(struct cache_fpl *fpl); | ||||
static int cache_fplookup_skip_slashes(struct cache_fpl *fpl); | static int cache_fplookup_skip_slashes(struct cache_fpl *fpl); | ||||
static int cache_fplookup_preparse(struct cache_fpl *fpl); | |||||
static void cache_fpl_pathlen_dec(struct cache_fpl *fpl); | static void cache_fpl_pathlen_dec(struct cache_fpl *fpl); | ||||
static void cache_fpl_pathlen_inc(struct cache_fpl *fpl); | static void cache_fpl_pathlen_inc(struct cache_fpl *fpl); | ||||
static void cache_fpl_pathlen_add(struct cache_fpl *fpl, size_t n); | |||||
static void cache_fpl_pathlen_sub(struct cache_fpl *fpl, size_t n); | static void cache_fpl_pathlen_sub(struct cache_fpl *fpl, size_t n); | ||||
static void | static void | ||||
cache_fpl_cleanup_cnp(struct componentname *cnp) | cache_fpl_cleanup_cnp(struct componentname *cnp) | ||||
{ | { | ||||
uma_zfree(namei_zone, cnp->cn_pnbuf); | uma_zfree(namei_zone, cnp->cn_pnbuf); | ||||
#ifdef DIAGNOSTIC | #ifdef DIAGNOSTIC | ||||
Show All 24 Lines | cache_fpl_handle_root(struct cache_fpl *fpl) | ||||
return (ndp->ni_rootdir); | return (ndp->ni_rootdir); | ||||
} | } | ||||
static void | static void | ||||
cache_fpl_checkpoint_outer(struct cache_fpl *fpl) | cache_fpl_checkpoint_outer(struct cache_fpl *fpl) | ||||
{ | { | ||||
fpl->snd_outer.ni_pathlen = fpl->ndp->ni_pathlen; | |||||
fpl->snd_outer.cn_flags = fpl->ndp->ni_cnd.cn_flags; | fpl->snd_outer.cn_flags = fpl->ndp->ni_cnd.cn_flags; | ||||
} | } | ||||
static void | static void | ||||
cache_fpl_checkpoint(struct cache_fpl *fpl) | cache_fpl_checkpoint(struct cache_fpl *fpl) | ||||
{ | { | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
Show All 17 Lines | |||||
{ | { | ||||
cache_fpl_restore_partial(fpl); | cache_fpl_restore_partial(fpl); | ||||
/* | /* | ||||
* It is 0 on entry by API contract. | * It is 0 on entry by API contract. | ||||
*/ | */ | ||||
fpl->ndp->ni_resflags = 0; | fpl->ndp->ni_resflags = 0; | ||||
fpl->ndp->ni_cnd.cn_nameptr = fpl->ndp->ni_cnd.cn_pnbuf; | fpl->ndp->ni_cnd.cn_nameptr = fpl->ndp->ni_cnd.cn_pnbuf; | ||||
fpl->ndp->ni_pathlen = fpl->snd_outer.ni_pathlen; | |||||
} | } | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
#define cache_fpl_smr_assert_entered(fpl) ({ \ | #define cache_fpl_smr_assert_entered(fpl) ({ \ | ||||
struct cache_fpl *_fpl = (fpl); \ | struct cache_fpl *_fpl = (fpl); \ | ||||
MPASS(_fpl->in_smr == true); \ | MPASS(_fpl->in_smr == true); \ | ||||
VFS_SMR_ASSERT_ENTERED(); \ | VFS_SMR_ASSERT_ENTERED(); \ | ||||
}) | }) | ||||
#define cache_fpl_smr_assert_not_entered(fpl) ({ \ | #define cache_fpl_smr_assert_not_entered(fpl) ({ \ | ||||
struct cache_fpl *_fpl = (fpl); \ | struct cache_fpl *_fpl = (fpl); \ | ||||
MPASS(_fpl->in_smr == false); \ | MPASS(_fpl->in_smr == false); \ | ||||
VFS_SMR_ASSERT_NOT_ENTERED(); \ | VFS_SMR_ASSERT_NOT_ENTERED(); \ | ||||
}) | }) | ||||
static void | static void | ||||
cache_fpl_assert_status(struct cache_fpl *fpl) | cache_fpl_assert_status(struct cache_fpl *fpl) | ||||
{ | { | ||||
switch (fpl->status) { | switch (fpl->status) { | ||||
case CACHE_FPL_STATUS_UNSET: | case CACHE_FPL_STATUS_UNSET: | ||||
__assert_unreachable(); | __assert_unreachable(); | ||||
break; | break; | ||||
case CACHE_FPL_STATUS_DESTROYED: | |||||
case CACHE_FPL_STATUS_ABORTED: | case CACHE_FPL_STATUS_ABORTED: | ||||
case CACHE_FPL_STATUS_PARTIAL: | case CACHE_FPL_STATUS_PARTIAL: | ||||
case CACHE_FPL_STATUS_HANDLED: | case CACHE_FPL_STATUS_HANDLED: | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
#else | #else | ||||
#define cache_fpl_smr_assert_entered(fpl) do { } while (0) | #define cache_fpl_smr_assert_entered(fpl) do { } while (0) | ||||
Show All 36 Lines | cache_fpl_aborted_early_impl(struct cache_fpl *fpl, int line) | ||||
return (CACHE_FPL_FAILED); | return (CACHE_FPL_FAILED); | ||||
} | } | ||||
#define cache_fpl_aborted_early(x) cache_fpl_aborted_early_impl((x), __LINE__) | #define cache_fpl_aborted_early(x) cache_fpl_aborted_early_impl((x), __LINE__) | ||||
static int __noinline | static int __noinline | ||||
cache_fpl_aborted_impl(struct cache_fpl *fpl, int line) | cache_fpl_aborted_impl(struct cache_fpl *fpl, int line) | ||||
{ | { | ||||
struct nameidata *ndp; | |||||
struct componentname *cnp; | |||||
ndp = fpl->ndp; | |||||
cnp = fpl->cnp; | |||||
if (fpl->status != CACHE_FPL_STATUS_UNSET) { | if (fpl->status != CACHE_FPL_STATUS_UNSET) { | ||||
KASSERT(fpl->status == CACHE_FPL_STATUS_PARTIAL, | KASSERT(fpl->status == CACHE_FPL_STATUS_PARTIAL, | ||||
("%s: converting to abort from %d at %d, set at %d\n", | ("%s: converting to abort from %d at %d, set at %d\n", | ||||
__func__, fpl->status, line, fpl->line)); | __func__, fpl->status, line, fpl->line)); | ||||
} | } | ||||
fpl->status = CACHE_FPL_STATUS_ABORTED; | fpl->status = CACHE_FPL_STATUS_ABORTED; | ||||
fpl->line = line; | fpl->line = line; | ||||
if (fpl->in_smr) | if (fpl->in_smr) | ||||
cache_fpl_smr_exit(fpl); | cache_fpl_smr_exit(fpl); | ||||
cache_fpl_restore_abort(fpl); | cache_fpl_restore_abort(fpl); | ||||
/* | |||||
* Resolving symlinks overwrites data passed by the caller. | |||||
* Let namei know. | |||||
*/ | |||||
if (ndp->ni_loopcnt > 0) { | |||||
fpl->status = CACHE_FPL_STATUS_DESTROYED; | |||||
cache_fpl_cleanup_cnp(cnp); | |||||
} | |||||
return (CACHE_FPL_FAILED); | return (CACHE_FPL_FAILED); | ||||
} | } | ||||
#define cache_fpl_aborted(x) cache_fpl_aborted_impl((x), __LINE__) | #define cache_fpl_aborted(x) cache_fpl_aborted_impl((x), __LINE__) | ||||
static int __noinline | static int __noinline | ||||
cache_fpl_partial_impl(struct cache_fpl *fpl, int line) | cache_fpl_partial_impl(struct cache_fpl *fpl, int line) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 124 Lines • ▼ Show 20 Lines | cache_fplookup_dirfd(struct cache_fpl *fpl, struct vnode **vpp) | ||||
error = fgetvp_lookup_smr(ndp->ni_dirfd, ndp, vpp, &fsearch); | error = fgetvp_lookup_smr(ndp->ni_dirfd, ndp, vpp, &fsearch); | ||||
if (__predict_false(error != 0)) { | if (__predict_false(error != 0)) { | ||||
return (cache_fpl_aborted(fpl)); | return (cache_fpl_aborted(fpl)); | ||||
} | } | ||||
fpl->fsearch = fsearch; | fpl->fsearch = fsearch; | ||||
return (0); | return (0); | ||||
} | } | ||||
static bool | |||||
cache_fplookup_vnode_supported(struct vnode *vp) | |||||
{ | |||||
return (vp->v_type != VLNK); | |||||
} | |||||
static int __noinline | static int __noinline | ||||
cache_fplookup_negative_promote(struct cache_fpl *fpl, struct namecache *oncp, | cache_fplookup_negative_promote(struct cache_fpl *fpl, struct namecache *oncp, | ||||
uint32_t hash) | uint32_t hash) | ||||
{ | { | ||||
struct componentname *cnp; | struct componentname *cnp; | ||||
struct vnode *dvp; | struct vnode *dvp; | ||||
cnp = fpl->cnp; | cnp = fpl->cnp; | ||||
▲ Show 20 Lines • Show All 264 Lines • ▼ Show 20 Lines | if (__predict_false(tvp == dvp)) { | ||||
return (cache_fpl_aborted(fpl)); | return (cache_fpl_aborted(fpl)); | ||||
} | } | ||||
/* | /* | ||||
* Check if the target is either a symlink or a mount point. | * Check if the target is either a symlink or a mount point. | ||||
* Since we expect this to be the terminal vnode it should | * Since we expect this to be the terminal vnode it should | ||||
* almost never be true. | * almost never be true. | ||||
*/ | */ | ||||
if (__predict_false(!cache_fplookup_vnode_supported(tvp) || | if (__predict_false(tvp->v_type == VLNK || cache_fplookup_is_mp(fpl))) { | ||||
cache_fplookup_is_mp(fpl))) { | |||||
vput(dvp); | vput(dvp); | ||||
vput(tvp); | vput(tvp); | ||||
return (cache_fpl_aborted(fpl)); | return (cache_fpl_aborted(fpl)); | ||||
} | } | ||||
if ((cnp->cn_flags & FAILIFEXISTS) != 0) { | if ((cnp->cn_flags & FAILIFEXISTS) != 0) { | ||||
vput(dvp); | vput(dvp); | ||||
vput(tvp); | vput(tvp); | ||||
▲ Show 20 Lines • Show All 286 Lines • ▼ Show 20 Lines | if (tvp == NULL) { | ||||
if ((cnp->cn_flags & (WANTPARENT | LOCKPARENT)) == 0) { | if ((cnp->cn_flags & (WANTPARENT | LOCKPARENT)) == 0) { | ||||
vput(dvp); | vput(dvp); | ||||
} else if ((cnp->cn_flags & LOCKPARENT) == 0) { | } else if ((cnp->cn_flags & LOCKPARENT) == 0) { | ||||
VOP_UNLOCK(dvp); | VOP_UNLOCK(dvp); | ||||
} | } | ||||
return (cache_fpl_handled(fpl)); | return (cache_fpl_handled(fpl)); | ||||
} | } | ||||
if (__predict_false(!cache_fplookup_vnode_supported(tvp) || | if (__predict_false(tvp->v_type == VLNK || cache_fplookup_is_mp(fpl))) { | ||||
cache_fplookup_is_mp(fpl))) { | |||||
vput(dvp); | vput(dvp); | ||||
vput(tvp); | vput(tvp); | ||||
return (cache_fpl_aborted(fpl)); | return (cache_fpl_aborted(fpl)); | ||||
} | } | ||||
if ((cnp->cn_flags & LOCKLEAF) == 0) { | if ((cnp->cn_flags & LOCKLEAF) == 0) { | ||||
VOP_UNLOCK(tvp); | VOP_UNLOCK(tvp); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 124 Lines • ▼ Show 20 Lines | cache_fplookup_neg(struct cache_fpl *fpl, struct namecache *ncp, uint32_t hash) | ||||
if (neg_promote) { | if (neg_promote) { | ||||
return (cache_fplookup_negative_promote(fpl, ncp, hash)); | return (cache_fplookup_negative_promote(fpl, ncp, hash)); | ||||
} | } | ||||
cache_neg_hit_finish(ncp); | cache_neg_hit_finish(ncp); | ||||
cache_fpl_smr_exit(fpl); | cache_fpl_smr_exit(fpl); | ||||
return (cache_fpl_handled_error(fpl, ENOENT)); | return (cache_fpl_handled_error(fpl, ENOENT)); | ||||
} | } | ||||
/* | |||||
* Resolve a symlink. Called by filesystem-specific routines. | |||||
* | |||||
* Code flow is: | |||||
* ... -> cache_fplookup_symlink -> VOP_FPLOOKUP_SYMLINK -> cache_symlink_resolve | |||||
*/ | |||||
int | |||||
cache_symlink_resolve(struct cache_fpl *fpl, const char *string, size_t len) | |||||
{ | |||||
struct nameidata *ndp; | |||||
struct componentname *cnp; | |||||
ndp = fpl->ndp; | |||||
cnp = fpl->cnp; | |||||
if (__predict_false(len == 0)) { | |||||
return (ENOENT); | |||||
} | |||||
ndp->ni_pathlen = fpl->nulchar - cnp->cn_nameptr - cnp->cn_namelen + 1; | |||||
#ifdef INVARIANTS | |||||
if (ndp->ni_pathlen != fpl->debug.ni_pathlen) { | |||||
panic("%s: mismatch (%zu != %zu) nulchar %p nameptr %p [%s] ; full string [%s]\n", | |||||
__func__, ndp->ni_pathlen, fpl->debug.ni_pathlen, fpl->nulchar, | |||||
cnp->cn_nameptr, cnp->cn_nameptr, cnp->cn_pnbuf); | |||||
} | |||||
#endif | |||||
if (__predict_false(len + ndp->ni_pathlen > MAXPATHLEN)) { | |||||
return (ENAMETOOLONG); | |||||
} | |||||
if (__predict_false(ndp->ni_loopcnt++ >= MAXSYMLINKS)) { | |||||
return (ELOOP); | |||||
} | |||||
if (ndp->ni_pathlen > 1) { | |||||
bcopy(ndp->ni_next, cnp->cn_pnbuf + len, ndp->ni_pathlen); | |||||
} else { | |||||
cnp->cn_pnbuf[len] = '\0'; | |||||
} | |||||
bcopy(string, cnp->cn_pnbuf, len); | |||||
ndp->ni_pathlen += len; | |||||
cache_fpl_pathlen_add(fpl, len); | |||||
cnp->cn_nameptr = cnp->cn_pnbuf; | |||||
fpl->nulchar = &cnp->cn_nameptr[ndp->ni_pathlen - 1]; | |||||
return (0); | |||||
} | |||||
static int __noinline | |||||
cache_fplookup_symlink(struct cache_fpl *fpl) | |||||
{ | |||||
struct nameidata *ndp; | |||||
struct componentname *cnp; | |||||
struct vnode *dvp, *tvp; | |||||
int error; | |||||
ndp = fpl->ndp; | |||||
cnp = fpl->cnp; | |||||
dvp = fpl->dvp; | |||||
tvp = fpl->tvp; | |||||
if (cache_fpl_islastcn(ndp)) { | |||||
if ((cnp->cn_flags & FOLLOW) == 0) { | |||||
return (cache_fplookup_final(fpl)); | |||||
} | |||||
} | |||||
error = VOP_FPLOOKUP_SYMLINK(tvp, fpl); | |||||
if (__predict_false(error != 0)) { | |||||
switch (error) { | |||||
case EAGAIN: | |||||
return (cache_fpl_partial(fpl)); | |||||
case ENOENT: | |||||
case ENAMETOOLONG: | |||||
case ELOOP: | |||||
cache_fpl_smr_exit(fpl); | |||||
return (cache_fpl_handled_error(fpl, error)); | |||||
default: | |||||
return (cache_fpl_aborted(fpl)); | |||||
} | |||||
} | |||||
if (*(cnp->cn_nameptr) == '/') { | |||||
fpl->dvp = cache_fpl_handle_root(fpl); | |||||
fpl->dvp_seqc = vn_seqc_read_any(fpl->dvp); | |||||
if (seqc_in_modify(fpl->dvp_seqc)) { | |||||
return (cache_fpl_aborted(fpl)); | |||||
} | |||||
} | |||||
return (cache_fplookup_preparse(fpl)); | |||||
} | |||||
static int | static int | ||||
cache_fplookup_next(struct cache_fpl *fpl) | cache_fplookup_next(struct cache_fpl *fpl) | ||||
{ | { | ||||
struct componentname *cnp; | struct componentname *cnp; | ||||
struct namecache *ncp; | struct namecache *ncp; | ||||
struct vnode *dvp, *tvp; | struct vnode *dvp, *tvp; | ||||
u_char nc_flag; | u_char nc_flag; | ||||
uint32_t hash; | uint32_t hash; | ||||
Show All 39 Lines | cache_fplookup_next(struct cache_fpl *fpl) | ||||
} | } | ||||
fpl->tvp = tvp; | fpl->tvp = tvp; | ||||
fpl->tvp_seqc = vn_seqc_read_any(tvp); | fpl->tvp_seqc = vn_seqc_read_any(tvp); | ||||
if (seqc_in_modify(fpl->tvp_seqc)) { | if (seqc_in_modify(fpl->tvp_seqc)) { | ||||
return (cache_fpl_partial(fpl)); | return (cache_fpl_partial(fpl)); | ||||
} | } | ||||
if (!cache_fplookup_vnode_supported(tvp)) { | |||||
return (cache_fpl_partial(fpl)); | |||||
} | |||||
counter_u64_add(numposhits, 1); | counter_u64_add(numposhits, 1); | ||||
SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name, tvp); | SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name, tvp); | ||||
error = 0; | error = 0; | ||||
if (cache_fplookup_is_mp(fpl)) { | if (cache_fplookup_is_mp(fpl)) { | ||||
error = cache_fplookup_cross_mount(fpl); | error = cache_fplookup_cross_mount(fpl); | ||||
} | } | ||||
return (error); | return (error); | ||||
▲ Show 20 Lines • Show All 168 Lines • ▼ Show 20 Lines | cache_fpl_pathlen_dec(struct cache_fpl *fpl) | ||||
cache_fpl_pathlen_sub(fpl, 1); | cache_fpl_pathlen_sub(fpl, 1); | ||||
} | } | ||||
static void | static void | ||||
cache_fpl_pathlen_inc(struct cache_fpl *fpl) | cache_fpl_pathlen_inc(struct cache_fpl *fpl) | ||||
{ | { | ||||
fpl->debug.ni_pathlen++; | cache_fpl_pathlen_add(fpl, 1); | ||||
} | } | ||||
static void | static void | ||||
cache_fpl_pathlen_add(struct cache_fpl *fpl, size_t n) | |||||
{ | |||||
fpl->debug.ni_pathlen += n; | |||||
KASSERT(fpl->debug.ni_pathlen <= PATH_MAX, | |||||
("%s: pathlen overflow to %zd\n", __func__, fpl->debug.ni_pathlen)); | |||||
} | |||||
static void | |||||
cache_fpl_pathlen_sub(struct cache_fpl *fpl, size_t n) | cache_fpl_pathlen_sub(struct cache_fpl *fpl, size_t n) | ||||
{ | { | ||||
fpl->debug.ni_pathlen -= n; | fpl->debug.ni_pathlen -= n; | ||||
KASSERT(fpl->debug.ni_pathlen <= PATH_MAX, | KASSERT(fpl->debug.ni_pathlen <= PATH_MAX, | ||||
("%s: pathlen underflow to %zd\n", __func__, fpl->debug.ni_pathlen)); | ("%s: pathlen underflow to %zd\n", __func__, fpl->debug.ni_pathlen)); | ||||
} | } | ||||
#else | #else | ||||
static void __always_inline | static void __always_inline | ||||
cache_fpl_pathlen_dec(struct cache_fpl *fpl) | cache_fpl_pathlen_dec(struct cache_fpl *fpl) | ||||
{ | { | ||||
} | } | ||||
static void __always_inline | static void __always_inline | ||||
cache_fpl_pathlen_inc(struct cache_fpl *fpl) | cache_fpl_pathlen_inc(struct cache_fpl *fpl) | ||||
{ | { | ||||
} | } | ||||
static void | static void | ||||
cache_fpl_pathlen_add(struct cache_fpl *fpl, size_t n) | |||||
{ | |||||
} | |||||
static void | |||||
cache_fpl_pathlen_sub(struct cache_fpl *fpl, size_t n) | cache_fpl_pathlen_sub(struct cache_fpl *fpl, size_t n) | ||||
{ | { | ||||
} | } | ||||
#endif | #endif | ||||
static int | static int __always_inline | ||||
cache_fplookup_preparse(struct cache_fpl *fpl) | cache_fplookup_preparse(struct cache_fpl *fpl) | ||||
{ | { | ||||
struct componentname *cnp; | struct componentname *cnp; | ||||
cnp = fpl->cnp; | cnp = fpl->cnp; | ||||
if (__predict_false(cnp->cn_nameptr[0] == '\0')) { | if (__predict_false(cnp->cn_nameptr[0] == '\0')) { | ||||
return (cache_fplookup_degenerate(fpl)); | return (cache_fplookup_degenerate(fpl)); | ||||
▲ Show 20 Lines • Show All 262 Lines • ▼ Show 20 Lines | cache_fplookup_impl(struct vnode *dvp, struct cache_fpl *fpl) | ||||
fpl->dvp = dvp; | fpl->dvp = dvp; | ||||
fpl->dvp_seqc = vn_seqc_read_notmodify(fpl->dvp); | fpl->dvp_seqc = vn_seqc_read_notmodify(fpl->dvp); | ||||
mp = atomic_load_ptr(&dvp->v_mount); | mp = atomic_load_ptr(&dvp->v_mount); | ||||
if (__predict_false(mp == NULL || !cache_fplookup_mp_supported(mp))) { | if (__predict_false(mp == NULL || !cache_fplookup_mp_supported(mp))) { | ||||
return (cache_fpl_aborted(fpl)); | return (cache_fpl_aborted(fpl)); | ||||
} | } | ||||
VNPASS(cache_fplookup_vnode_supported(fpl->dvp), fpl->dvp); | |||||
error = cache_fplookup_preparse(fpl); | error = cache_fplookup_preparse(fpl); | ||||
if (__predict_false(cache_fpl_terminated(fpl))) { | if (__predict_false(cache_fpl_terminated(fpl))) { | ||||
return (error); | return (error); | ||||
} | } | ||||
for (;;) { | for (;;) { | ||||
error = cache_fplookup_parse(fpl); | error = cache_fplookup_parse(fpl); | ||||
if (__predict_false(error != 0)) { | if (__predict_false(error != 0)) { | ||||
break; | break; | ||||
} | } | ||||
VNPASS(cache_fplookup_vnode_supported(fpl->dvp), fpl->dvp); | |||||
error = VOP_FPLOOKUP_VEXEC(fpl->dvp, cnp->cn_cred); | error = VOP_FPLOOKUP_VEXEC(fpl->dvp, cnp->cn_cred); | ||||
if (__predict_false(error != 0)) { | if (__predict_false(error != 0)) { | ||||
error = cache_fplookup_failed_vexec(fpl, error); | error = cache_fplookup_failed_vexec(fpl, error); | ||||
break; | break; | ||||
} | } | ||||
error = cache_fplookup_next(fpl); | error = cache_fplookup_next(fpl); | ||||
if (__predict_false(cache_fpl_terminated(fpl))) { | if (__predict_false(cache_fpl_terminated(fpl))) { | ||||
break; | break; | ||||
} | } | ||||
VNPASS(!seqc_in_modify(fpl->tvp_seqc), fpl->tvp); | VNPASS(!seqc_in_modify(fpl->tvp_seqc), fpl->tvp); | ||||
if (fpl->tvp->v_type == VLNK) { | |||||
error = cache_fplookup_symlink(fpl); | |||||
if (cache_fpl_terminated(fpl)) { | |||||
break; | |||||
} | |||||
} else { | |||||
if (cache_fpl_islastcn(ndp)) { | if (cache_fpl_islastcn(ndp)) { | ||||
error = cache_fplookup_final(fpl); | error = cache_fplookup_final(fpl); | ||||
break; | break; | ||||
} | } | ||||
if (!vn_seqc_consistent(fpl->dvp, fpl->dvp_seqc)) { | if (!vn_seqc_consistent(fpl->dvp, fpl->dvp_seqc)) { | ||||
error = cache_fpl_aborted(fpl); | error = cache_fpl_aborted(fpl); | ||||
break; | break; | ||||
} | } | ||||
fpl->dvp = fpl->tvp; | fpl->dvp = fpl->tvp; | ||||
fpl->dvp_seqc = fpl->tvp_seqc; | fpl->dvp_seqc = fpl->tvp_seqc; | ||||
cache_fplookup_parse_advance(fpl); | cache_fplookup_parse_advance(fpl); | ||||
} | |||||
cache_fpl_checkpoint(fpl); | cache_fpl_checkpoint(fpl); | ||||
} | } | ||||
return (error); | return (error); | ||||
} | } | ||||
/* | /* | ||||
* Fast path lookup protected with SMR and sequence counters. | * Fast path lookup protected with SMR and sequence counters. | ||||
▲ Show 20 Lines • Show All 164 Lines • Show Last 20 Lines |