Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/vfs_cache.c
Show First 20 Lines • Show All 542 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* UMA zones. | * UMA zones. | ||||
*/ | */ | ||||
static uma_zone_t __read_mostly cache_zone_small; | static uma_zone_t __read_mostly cache_zone_small; | ||||
static uma_zone_t __read_mostly cache_zone_small_ts; | static uma_zone_t __read_mostly cache_zone_small_ts; | ||||
static uma_zone_t __read_mostly cache_zone_large; | static uma_zone_t __read_mostly cache_zone_large; | ||||
static uma_zone_t __read_mostly cache_zone_large_ts; | static uma_zone_t __read_mostly cache_zone_large_ts; | ||||
char * | |||||
cache_symlink_alloc(size_t size, int flags) | |||||
{ | |||||
if (size < CACHE_ZONE_SMALL_SIZE) { | |||||
return (uma_zalloc_smr(cache_zone_small, flags)); | |||||
} | |||||
if (size < CACHE_ZONE_LARGE_SIZE) { | |||||
return (uma_zalloc_smr(cache_zone_large, flags)); | |||||
} | |||||
return (NULL); | |||||
} | |||||
void | |||||
cache_symlink_free(char *string) | |||||
{ | |||||
size_t size; | |||||
MPASS(string != NULL); | |||||
/* | |||||
* XXXMJG | |||||
*/ | |||||
size = strlen(string); | |||||
if (size < CACHE_ZONE_SMALL_SIZE) { | |||||
uma_zfree_smr(cache_zone_small, string); | |||||
return; | |||||
} | |||||
if (size < CACHE_ZONE_LARGE_SIZE) { | |||||
uma_zfree_smr(cache_zone_large, string); | |||||
return; | |||||
} | |||||
__assert_unreachable(); | |||||
} | |||||
static struct namecache * | static struct namecache * | ||||
cache_alloc_uma(int len, bool ts) | cache_alloc_uma(int len, bool ts) | ||||
{ | { | ||||
struct namecache_ts *ncp_ts; | struct namecache_ts *ncp_ts; | ||||
struct namecache *ncp; | struct namecache *ncp; | ||||
if (__predict_false(ts)) { | if (__predict_false(ts)) { | ||||
if (len <= CACHE_PATH_CUTOFF) | if (len <= CACHE_PATH_CUTOFF) | ||||
▲ Show 20 Lines • Show All 3,158 Lines • ▼ Show 20 Lines | cache_fpl_handled_impl(struct cache_fpl *fpl, int error, int line) | ||||
MPASS(error != CACHE_FPL_FAILED); | MPASS(error != CACHE_FPL_FAILED); | ||||
fpl->status = CACHE_FPL_STATUS_HANDLED; | fpl->status = CACHE_FPL_STATUS_HANDLED; | ||||
fpl->line = line; | fpl->line = line; | ||||
return (error); | return (error); | ||||
} | } | ||||
#define cache_fpl_handled(x, e) cache_fpl_handled_impl((x), (e), __LINE__) | #define cache_fpl_handled(x, e) cache_fpl_handled_impl((x), (e), __LINE__) | ||||
static bool | |||||
cache_fpl_terminated(struct cache_fpl *fpl) | |||||
{ | |||||
return (fpl->status != CACHE_FPL_STATUS_UNSET); | |||||
} | |||||
#define CACHE_FPL_SUPPORTED_CN_FLAGS \ | #define CACHE_FPL_SUPPORTED_CN_FLAGS \ | ||||
(NC_NOMAKEENTRY | NC_KEEPPOSENTRY | LOCKLEAF | LOCKPARENT | WANTPARENT | \ | (NC_NOMAKEENTRY | NC_KEEPPOSENTRY | LOCKLEAF | LOCKPARENT | WANTPARENT | \ | ||||
FOLLOW | LOCKSHARED | SAVENAME | SAVESTART | WILLBEDIR | ISOPEN | \ | FOLLOW | LOCKSHARED | SAVENAME | SAVESTART | WILLBEDIR | ISOPEN | \ | ||||
NOMACCHECK | AUDITVNODE1 | AUDITVNODE2 | NOCAPCHECK) | NOMACCHECK | AUDITVNODE1 | AUDITVNODE2 | NOCAPCHECK) | ||||
#define CACHE_FPL_INTERNAL_CN_FLAGS \ | #define CACHE_FPL_INTERNAL_CN_FLAGS \ | ||||
(ISDOTDOT | MAKEENTRY | ISLASTCN) | (ISDOTDOT | MAKEENTRY | ISLASTCN) | ||||
▲ Show 20 Lines • Show All 69 Lines • ▼ Show 20 Lines | cache_fplookup_dirfd(struct cache_fpl *fpl, struct vnode **vpp) | ||||
if (__predict_false(error != 0)) { | if (__predict_false(error != 0)) { | ||||
cache_fpl_smr_exit(fpl); | cache_fpl_smr_exit(fpl); | ||||
return (cache_fpl_aborted(fpl)); | return (cache_fpl_aborted(fpl)); | ||||
} | } | ||||
fpl->fsearch = fsearch; | fpl->fsearch = fsearch; | ||||
return (0); | return (0); | ||||
} | } | ||||
static bool | |||||
cache_fplookup_vnode_supported(struct vnode *vp) | |||||
{ | |||||
return (vp->v_type != VLNK); | |||||
} | |||||
static int __noinline | static int __noinline | ||||
cache_fplookup_negative_promote(struct cache_fpl *fpl, struct namecache *oncp, | cache_fplookup_negative_promote(struct cache_fpl *fpl, struct namecache *oncp, | ||||
uint32_t hash) | uint32_t hash) | ||||
{ | { | ||||
struct componentname *cnp; | struct componentname *cnp; | ||||
struct vnode *dvp; | struct vnode *dvp; | ||||
cnp = fpl->cnp; | cnp = fpl->cnp; | ||||
▲ Show 20 Lines • Show All 180 Lines • ▼ Show 20 Lines | cache_fplookup_final(struct cache_fpl *fpl) | ||||
struct vnode *dvp, *tvp; | struct vnode *dvp, *tvp; | ||||
seqc_t dvp_seqc; | seqc_t dvp_seqc; | ||||
cnp = fpl->cnp; | cnp = fpl->cnp; | ||||
dvp = fpl->dvp; | dvp = fpl->dvp; | ||||
dvp_seqc = fpl->dvp_seqc; | dvp_seqc = fpl->dvp_seqc; | ||||
tvp = fpl->tvp; | tvp = fpl->tvp; | ||||
VNPASS(cache_fplookup_vnode_supported(dvp), dvp); | |||||
if (cnp->cn_nameiop != LOOKUP) { | if (cnp->cn_nameiop != LOOKUP) { | ||||
return (cache_fplookup_final_modifying(fpl)); | return (cache_fplookup_final_modifying(fpl)); | ||||
} | } | ||||
if ((cnp->cn_flags & (LOCKPARENT|WANTPARENT)) != 0) | if ((cnp->cn_flags & (LOCKPARENT|WANTPARENT)) != 0) | ||||
return (cache_fplookup_final_withparent(fpl)); | return (cache_fplookup_final_withparent(fpl)); | ||||
tvs = vget_prep_smr(tvp); | tvs = vget_prep_smr(tvp); | ||||
▲ Show 20 Lines • Show All 130 Lines • ▼ Show 20 Lines | cache_fplookup_neg(struct cache_fpl *fpl, struct namecache *ncp, uint32_t hash) | ||||
if (neg_promote) { | if (neg_promote) { | ||||
return (cache_fplookup_negative_promote(fpl, ncp, hash)); | return (cache_fplookup_negative_promote(fpl, ncp, hash)); | ||||
} | } | ||||
cache_neg_hit_finish(ncp); | cache_neg_hit_finish(ncp); | ||||
cache_fpl_smr_exit(fpl); | cache_fpl_smr_exit(fpl); | ||||
return (cache_fpl_handled(fpl, ENOENT)); | return (cache_fpl_handled(fpl, ENOENT)); | ||||
} | } | ||||
/* | |||||
* Resolve a symlink. Called by filesystem-specific routines. | |||||
* | |||||
* Code flow is: | |||||
* ... -> cache_fplookup_symlink -> VOP_FPLOOKUP_SYMLINK -> cache_symlink_resolve | |||||
*/ | |||||
int | |||||
cache_symlink_resolve(struct cache_fpl *fpl, const char *string, size_t len) | |||||
{ | |||||
struct nameidata *ndp; | |||||
struct componentname *cnp; | |||||
ndp = fpl->ndp; | |||||
cnp = fpl->cnp; | |||||
if (len == 0) { | |||||
return (ENOENT); | |||||
} | |||||
if (len + ndp->ni_pathlen > MAXPATHLEN) { | |||||
return (ENAMETOOLONG); | |||||
} | |||||
if (ndp->ni_loopcnt++ >= MAXSYMLINKS) { | |||||
return (ELOOP); | |||||
} | |||||
if (ndp->ni_pathlen > 1) { | |||||
bcopy(ndp->ni_next, cnp->cn_pnbuf + len, ndp->ni_pathlen); | |||||
} else { | |||||
cnp->cn_pnbuf[len] = '\0'; | |||||
} | |||||
bcopy(string, cnp->cn_pnbuf, len); | |||||
ndp->ni_pathlen += len; | |||||
cnp->cn_nameptr = cnp->cn_pnbuf; | |||||
return (0); | |||||
} | |||||
static int __noinline | |||||
cache_fplookup_symlink(struct cache_fpl *fpl) | |||||
{ | |||||
struct nameidata *ndp; | |||||
struct componentname *cnp; | |||||
struct vnode *dvp, *tvp; | |||||
int error; | |||||
ndp = fpl->ndp; | |||||
cnp = fpl->cnp; | |||||
dvp = fpl->dvp; | |||||
tvp = fpl->tvp; | |||||
if (cache_fpl_islastcn(ndp)) { | |||||
if ((cnp->cn_flags & FOLLOW) == 0) { | |||||
return (cache_fplookup_final(fpl)); | |||||
} | |||||
} | |||||
error = VOP_FPLOOKUP_SYMLINK(tvp, fpl); | |||||
if (__predict_false(error != 0)) { | |||||
switch (error) { | |||||
case EAGAIN: | |||||
return (cache_fpl_partial(fpl)); | |||||
case ENOENT: | |||||
case ENAMETOOLONG: | |||||
case ELOOP: | |||||
cache_fpl_smr_exit(fpl); | |||||
return (cache_fpl_handled(fpl, error)); | |||||
default: | |||||
return (cache_fpl_aborted(fpl)); | |||||
} | |||||
} | |||||
if (*(cnp->cn_nameptr) == '/') { | |||||
cache_fpl_handle_root(ndp, &fpl->dvp); | |||||
fpl->dvp_seqc = vn_seqc_read_any(tvp); | |||||
if (seqc_in_modify(fpl->dvp_seqc)) { | |||||
return (cache_fpl_aborted(fpl)); | |||||
} | |||||
} | |||||
return (0); | |||||
} | |||||
static int | static int | ||||
cache_fplookup_next(struct cache_fpl *fpl) | cache_fplookup_next(struct cache_fpl *fpl) | ||||
{ | { | ||||
struct componentname *cnp; | struct componentname *cnp; | ||||
struct namecache *ncp; | struct namecache *ncp; | ||||
struct vnode *dvp, *tvp; | struct vnode *dvp, *tvp; | ||||
u_char nc_flag; | u_char nc_flag; | ||||
uint32_t hash; | uint32_t hash; | ||||
Show All 33 Lines | cache_fplookup_next(struct cache_fpl *fpl) | ||||
} | } | ||||
fpl->tvp = tvp; | fpl->tvp = tvp; | ||||
fpl->tvp_seqc = vn_seqc_read_any(tvp); | fpl->tvp_seqc = vn_seqc_read_any(tvp); | ||||
if (seqc_in_modify(fpl->tvp_seqc)) { | if (seqc_in_modify(fpl->tvp_seqc)) { | ||||
return (cache_fpl_partial(fpl)); | return (cache_fpl_partial(fpl)); | ||||
} | } | ||||
if (!cache_fplookup_vnode_supported(tvp)) { | |||||
return (cache_fpl_partial(fpl)); | |||||
} | |||||
counter_u64_add(numposhits, 1); | counter_u64_add(numposhits, 1); | ||||
SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name, tvp); | SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name, tvp); | ||||
return (0); | return (0); | ||||
} | } | ||||
static bool | static bool | ||||
cache_fplookup_mp_supported(struct mount *mp) | cache_fplookup_mp_supported(struct mount *mp) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 292 Lines • ▼ Show 20 Lines | if (seqc_in_modify(fpl->dvp_seqc)) { | ||||
goto out; | goto out; | ||||
} | } | ||||
mp = atomic_load_ptr(&fpl->dvp->v_mount); | mp = atomic_load_ptr(&fpl->dvp->v_mount); | ||||
if (!cache_fplookup_mp_supported(mp)) { | if (!cache_fplookup_mp_supported(mp)) { | ||||
cache_fpl_aborted(fpl); | cache_fpl_aborted(fpl); | ||||
goto out; | goto out; | ||||
} | } | ||||
VNPASS(cache_fplookup_vnode_supported(fpl->dvp), fpl->dvp); | |||||
for (;;) { | for (;;) { | ||||
error = cache_fplookup_parse(fpl); | error = cache_fplookup_parse(fpl); | ||||
if (__predict_false(error != 0)) { | if (__predict_false(error != 0)) { | ||||
break; | break; | ||||
} | } | ||||
VNPASS(cache_fplookup_vnode_supported(fpl->dvp), fpl->dvp); | |||||
error = VOP_FPLOOKUP_VEXEC(fpl->dvp, cnp->cn_cred); | error = VOP_FPLOOKUP_VEXEC(fpl->dvp, cnp->cn_cred); | ||||
if (__predict_false(error != 0)) { | if (__predict_false(error != 0)) { | ||||
error = cache_fplookup_failed_vexec(fpl, error); | error = cache_fplookup_failed_vexec(fpl, error); | ||||
break; | break; | ||||
} | } | ||||
if (__predict_false(cache_fpl_isdotdot(cnp))) { | if (__predict_false(cache_fpl_isdotdot(cnp))) { | ||||
error = cache_fplookup_dotdot(fpl); | error = cache_fplookup_dotdot(fpl); | ||||
Show All 13 Lines | if (__predict_false(cache_fpl_isdotdot(cnp))) { | ||||
if (__predict_false(error != 0)) { | if (__predict_false(error != 0)) { | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
} | } | ||||
VNPASS(!seqc_in_modify(fpl->tvp_seqc), fpl->tvp); | VNPASS(!seqc_in_modify(fpl->tvp_seqc), fpl->tvp); | ||||
if (fpl->tvp->v_type == VLNK) { | |||||
error = cache_fplookup_symlink(fpl); | |||||
if (cache_fpl_terminated(fpl)) { | |||||
break; | |||||
} | |||||
} else { | |||||
if (cache_fpl_islastcn(ndp)) { | if (cache_fpl_islastcn(ndp)) { | ||||
error = cache_fplookup_final(fpl); | error = cache_fplookup_final(fpl); | ||||
break; | break; | ||||
} | } | ||||
if (!vn_seqc_consistent(fpl->dvp, fpl->dvp_seqc)) { | if (!vn_seqc_consistent(fpl->dvp, fpl->dvp_seqc)) { | ||||
error = cache_fpl_aborted(fpl); | error = cache_fpl_aborted(fpl); | ||||
break; | break; | ||||
} | } | ||||
fpl->dvp = fpl->tvp; | fpl->dvp = fpl->tvp; | ||||
fpl->dvp_seqc = fpl->tvp_seqc; | fpl->dvp_seqc = fpl->tvp_seqc; | ||||
cache_fplookup_parse_advance(fpl); | cache_fplookup_parse_advance(fpl); | ||||
} | |||||
VNPASS(!seqc_in_modify(fpl->dvp_seqc), fpl->dvp); | |||||
cache_fpl_checkpoint(fpl, &fpl->snd); | cache_fpl_checkpoint(fpl, &fpl->snd); | ||||
} | } | ||||
out: | out: | ||||
switch (fpl->status) { | switch (fpl->status) { | ||||
case CACHE_FPL_STATUS_DESTROYED: | |||||
case CACHE_FPL_STATUS_UNSET: | case CACHE_FPL_STATUS_UNSET: | ||||
__assert_unreachable(); | __assert_unreachable(); | ||||
break; | break; | ||||
case CACHE_FPL_STATUS_PARTIAL: | case CACHE_FPL_STATUS_PARTIAL: | ||||
cache_fpl_smr_assert_entered(fpl); | cache_fpl_smr_assert_entered(fpl); | ||||
return (cache_fplookup_partial_setup(fpl)); | return (cache_fplookup_partial_setup(fpl)); | ||||
case CACHE_FPL_STATUS_ABORTED: | case CACHE_FPL_STATUS_ABORTED: | ||||
if (fpl->in_smr) | if (fpl->in_smr) | ||||
▲ Show 20 Lines • Show All 146 Lines • ▼ Show 20 Lines | if (cnp->cn_pnbuf[0] == '/') { | ||||
} | } | ||||
} | } | ||||
SDT_PROBE4(vfs, namei, lookup, entry, dvp, cnp->cn_pnbuf, cnp->cn_flags, true); | SDT_PROBE4(vfs, namei, lookup, entry, dvp, cnp->cn_pnbuf, cnp->cn_flags, true); | ||||
error = cache_fplookup_impl(dvp, &fpl); | error = cache_fplookup_impl(dvp, &fpl); | ||||
out: | out: | ||||
cache_fpl_smr_assert_not_entered(&fpl); | cache_fpl_smr_assert_not_entered(&fpl); | ||||
SDT_PROBE3(vfs, fplookup, lookup, done, ndp, fpl.line, fpl.status); | |||||
*status = fpl.status; | |||||
switch (fpl.status) { | switch (fpl.status) { | ||||
case CACHE_FPL_STATUS_DESTROYED: | |||||
case CACHE_FPL_STATUS_UNSET: | case CACHE_FPL_STATUS_UNSET: | ||||
__assert_unreachable(); | __assert_unreachable(); | ||||
break; | break; | ||||
case CACHE_FPL_STATUS_HANDLED: | case CACHE_FPL_STATUS_HANDLED: | ||||
SDT_PROBE3(vfs, namei, lookup, return, error, | SDT_PROBE3(vfs, namei, lookup, return, error, | ||||
(error == 0 ? ndp->ni_vp : NULL), true); | (error == 0 ? ndp->ni_vp : NULL), true); | ||||
break; | break; | ||||
case CACHE_FPL_STATUS_PARTIAL: | case CACHE_FPL_STATUS_PARTIAL: | ||||
*pwdp = fpl.pwd; | *pwdp = fpl.pwd; | ||||
/* | /* | ||||
* Status restored by cache_fplookup_partial_setup. | * Status restored by cache_fplookup_partial_setup. | ||||
*/ | */ | ||||
break; | break; | ||||
case CACHE_FPL_STATUS_ABORTED: | case CACHE_FPL_STATUS_ABORTED: | ||||
cache_fpl_restore_abort(&fpl, &orig); | cache_fpl_restore_abort(&fpl, &orig); | ||||
/* | |||||
* Resolving symlinks overwrites data passed by the caller. | |||||
* Let namei know. | |||||
*/ | |||||
if (ndp->ni_loopcnt > 0) { | |||||
fpl.status = CACHE_FPL_STATUS_DESTROYED; | |||||
cache_fpl_cleanup_cnp(cnp); | |||||
} | |||||
break; | break; | ||||
} | } | ||||
SDT_PROBE3(vfs, fplookup, lookup, done, ndp, fpl.line, fpl.status); | |||||
*status = fpl.status; | |||||
return (error); | return (error); | ||||
} | } |