Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/vfs_cache.c
Show First 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | |||||
#include <sys/lock.h> | #include <sys/lock.h> | ||||
#include <sys/malloc.h> | #include <sys/malloc.h> | ||||
#include <sys/fcntl.h> | #include <sys/fcntl.h> | ||||
#include <sys/mount.h> | #include <sys/mount.h> | ||||
#include <sys/namei.h> | #include <sys/namei.h> | ||||
#include <sys/proc.h> | #include <sys/proc.h> | ||||
#include <sys/rwlock.h> | #include <sys/rwlock.h> | ||||
#include <sys/sdt.h> | #include <sys/sdt.h> | ||||
#include <sys/smr.h> | |||||
#include <sys/smp.h> | #include <sys/smp.h> | ||||
#include <sys/syscallsubr.h> | #include <sys/syscallsubr.h> | ||||
#include <sys/sysctl.h> | #include <sys/sysctl.h> | ||||
#include <sys/sysproto.h> | #include <sys/sysproto.h> | ||||
#include <sys/vnode.h> | #include <sys/vnode.h> | ||||
#include <ck_queue.h> | |||||
#ifdef KTRACE | #ifdef KTRACE | ||||
#include <sys/ktrace.h> | #include <sys/ktrace.h> | ||||
#endif | #endif | ||||
#ifdef DDB | #ifdef DDB | ||||
#include <ddb/ddb.h> | #include <ddb/ddb.h> | ||||
#endif | #endif | ||||
Show All 27 Lines | SDT_PROBE_DEFINE2(vfs, namecache, shrink_negative, done, "struct vnode *", | ||||
"char *"); | "char *"); | ||||
/* | /* | ||||
* This structure describes the elements in the cache of recent | * This structure describes the elements in the cache of recent | ||||
* names looked up by namei. | * names looked up by namei. | ||||
*/ | */ | ||||
struct namecache { | struct namecache { | ||||
LIST_ENTRY(namecache) nc_hash; /* hash chain */ | CK_LIST_ENTRY(namecache) nc_hash;/* hash chain */ | ||||
LIST_ENTRY(namecache) nc_src; /* source vnode list */ | LIST_ENTRY(namecache) nc_src; /* source vnode list */ | ||||
TAILQ_ENTRY(namecache) nc_dst; /* destination vnode list */ | TAILQ_ENTRY(namecache) nc_dst; /* destination vnode list */ | ||||
struct vnode *nc_dvp; /* vnode of parent of name */ | struct vnode *nc_dvp; /* vnode of parent of name */ | ||||
union { | union { | ||||
struct vnode *nu_vp; /* vnode the name refers to */ | struct vnode *nu_vp; /* vnode the name refers to */ | ||||
} n_un; | } n_un; | ||||
u_char nc_flag; /* flag bits */ | u_char nc_flag; /* flag bits */ | ||||
u_char nc_nlen; /* length of name */ | u_char nc_nlen; /* length of name */ | ||||
Show All 22 Lines | |||||
*/ | */ | ||||
#define NCF_WHITE 0x01 | #define NCF_WHITE 0x01 | ||||
#define NCF_ISDOTDOT 0x02 | #define NCF_ISDOTDOT 0x02 | ||||
#define NCF_TS 0x04 | #define NCF_TS 0x04 | ||||
#define NCF_DTS 0x08 | #define NCF_DTS 0x08 | ||||
#define NCF_DVDROP 0x10 | #define NCF_DVDROP 0x10 | ||||
#define NCF_NEGATIVE 0x20 | #define NCF_NEGATIVE 0x20 | ||||
#define NCF_HOTNEGATIVE 0x40 | #define NCF_HOTNEGATIVE 0x40 | ||||
#define NCF_INVALID 0x80 | |||||
static bool | |||||
cache_ncp_invalid(struct namecache *ncp) | |||||
{ | |||||
atomic_thread_fence_acq(); | |||||
return ((ncp->nc_flag & NCF_INVALID) != 0); | |||||
} | |||||
static void | |||||
cache_ncp_invalidate(struct namecache *ncp) | |||||
{ | |||||
atomic_thread_fence_rel(); | |||||
KASSERT((ncp->nc_flag & NCF_INVALID) == 0, | |||||
("%s: entry %p already invalid", __func__, ncp)); | |||||
ncp->nc_flag |= NCF_INVALID; | |||||
} | |||||
/* | /* | ||||
* Name caching works as follows: | * Name caching works as follows: | ||||
* | * | ||||
* Names found by directory scans are retained in a cache | * Names found by directory scans are retained in a cache | ||||
* for future reference. It is managed LRU, so frequently | * for future reference. It is managed LRU, so frequently | ||||
* used names will hang around. Cache is indexed by hash value | * used names will hang around. Cache is indexed by hash value | ||||
* obtained from (dvp, name) where dvp refers to the directory | * obtained from (dvp, name) where dvp refers to the directory | ||||
* containing name. | * containing name. | ||||
Show All 32 Lines | |||||
* threads can purge two different vnodes and try to remove the same name. | * threads can purge two different vnodes and try to remove the same name. | ||||
* | * | ||||
* If the already held vnode lock is lower than the second required lock, we | * If the already held vnode lock is lower than the second required lock, we | ||||
* can just take the other lock. However, in the opposite case, this could | * can just take the other lock. However, in the opposite case, this could | ||||
* deadlock. As such, this is resolved by trylocking and if that fails unlocking | * deadlock. As such, this is resolved by trylocking and if that fails unlocking | ||||
* the first node, locking everything in order and revalidating the state. | * the first node, locking everything in order and revalidating the state. | ||||
*/ | */ | ||||
VFS_SMR_DECLARE; | |||||
/* | /* | ||||
* Structures associated with name caching. | * Structures associated with name caching. | ||||
*/ | */ | ||||
#define NCHHASH(hash) \ | #define NCHHASH(hash) \ | ||||
(&nchashtbl[(hash) & nchash]) | (&nchashtbl[(hash) & nchash]) | ||||
static __read_mostly LIST_HEAD(nchashhead, namecache) *nchashtbl;/* Hash Table */ | static __read_mostly CK_LIST_HEAD(nchashhead, namecache) *nchashtbl;/* Hash Table */ | ||||
static u_long __read_mostly nchash; /* size of hash table */ | static u_long __read_mostly nchash; /* size of hash table */ | ||||
SYSCTL_ULONG(_debug, OID_AUTO, nchash, CTLFLAG_RD, &nchash, 0, | SYSCTL_ULONG(_debug, OID_AUTO, nchash, CTLFLAG_RD, &nchash, 0, | ||||
"Size of namecache hash table"); | "Size of namecache hash table"); | ||||
static u_long __read_mostly ncnegfactor = 5; /* ratio of negative entries */ | static u_long __read_mostly ncnegfactor = 5; /* ratio of negative entries */ | ||||
SYSCTL_ULONG(_vfs, OID_AUTO, ncnegfactor, CTLFLAG_RW, &ncnegfactor, 0, | SYSCTL_ULONG(_vfs, OID_AUTO, ncnegfactor, CTLFLAG_RW, &ncnegfactor, 0, | ||||
"Ratio of negative namecache entries"); | "Ratio of negative namecache entries"); | ||||
static u_long __exclusive_cache_line numneg; /* number of negative entries allocated */ | static u_long __exclusive_cache_line numneg; /* number of negative entries allocated */ | ||||
static u_long __exclusive_cache_line numcache;/* number of cache entries allocated */ | static u_long __exclusive_cache_line numcache;/* number of cache entries allocated */ | ||||
▲ Show 20 Lines • Show All 61 Lines • ▼ Show 20 Lines | |||||
static struct namecache * | static struct namecache * | ||||
cache_alloc(int len, int ts) | cache_alloc(int len, int ts) | ||||
{ | { | ||||
struct namecache_ts *ncp_ts; | struct namecache_ts *ncp_ts; | ||||
struct namecache *ncp; | struct namecache *ncp; | ||||
if (__predict_false(ts)) { | if (__predict_false(ts)) { | ||||
if (len <= CACHE_PATH_CUTOFF) | if (len <= CACHE_PATH_CUTOFF) | ||||
ncp_ts = uma_zalloc(cache_zone_small_ts, M_WAITOK); | ncp_ts = uma_zalloc_smr(cache_zone_small_ts, M_WAITOK); | ||||
else | else | ||||
ncp_ts = uma_zalloc(cache_zone_large_ts, M_WAITOK); | ncp_ts = uma_zalloc_smr(cache_zone_large_ts, M_WAITOK); | ||||
ncp = &ncp_ts->nc_nc; | ncp = &ncp_ts->nc_nc; | ||||
} else { | } else { | ||||
if (len <= CACHE_PATH_CUTOFF) | if (len <= CACHE_PATH_CUTOFF) | ||||
ncp = uma_zalloc(cache_zone_small, M_WAITOK); | ncp = uma_zalloc_smr(cache_zone_small, M_WAITOK); | ||||
else | else | ||||
ncp = uma_zalloc(cache_zone_large, M_WAITOK); | ncp = uma_zalloc_smr(cache_zone_large, M_WAITOK); | ||||
} | } | ||||
return (ncp); | return (ncp); | ||||
} | } | ||||
static void | static void | ||||
cache_free(struct namecache *ncp) | cache_free(struct namecache *ncp) | ||||
{ | { | ||||
struct namecache_ts *ncp_ts; | struct namecache_ts *ncp_ts; | ||||
if (ncp == NULL) | if (ncp == NULL) | ||||
return; | return; | ||||
if ((ncp->nc_flag & NCF_DVDROP) != 0) | if ((ncp->nc_flag & NCF_DVDROP) != 0) | ||||
vdrop(ncp->nc_dvp); | vdrop(ncp->nc_dvp); | ||||
if (__predict_false(ncp->nc_flag & NCF_TS)) { | if (__predict_false(ncp->nc_flag & NCF_TS)) { | ||||
ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc); | ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc); | ||||
if (ncp->nc_nlen <= CACHE_PATH_CUTOFF) | if (ncp->nc_nlen <= CACHE_PATH_CUTOFF) | ||||
uma_zfree(cache_zone_small_ts, ncp_ts); | uma_zfree_smr(cache_zone_small_ts, ncp_ts); | ||||
else | else | ||||
uma_zfree(cache_zone_large_ts, ncp_ts); | uma_zfree_smr(cache_zone_large_ts, ncp_ts); | ||||
} else { | } else { | ||||
if (ncp->nc_nlen <= CACHE_PATH_CUTOFF) | if (ncp->nc_nlen <= CACHE_PATH_CUTOFF) | ||||
uma_zfree(cache_zone_small, ncp); | uma_zfree_smr(cache_zone_small, ncp); | ||||
else | else | ||||
uma_zfree(cache_zone_large, ncp); | uma_zfree_smr(cache_zone_large, ncp); | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
cache_out_ts(struct namecache *ncp, struct timespec *tsp, int *ticksp) | cache_out_ts(struct namecache *ncp, struct timespec *tsp, int *ticksp) | ||||
{ | { | ||||
struct namecache_ts *ncp_ts; | struct namecache_ts *ncp_ts; | ||||
Show All 12 Lines | |||||
} | } | ||||
#ifdef DEBUG_CACHE | #ifdef DEBUG_CACHE | ||||
static int __read_mostly doingcache = 1; /* 1 => enable the cache */ | static int __read_mostly doingcache = 1; /* 1 => enable the cache */ | ||||
SYSCTL_INT(_debug, OID_AUTO, vfscache, CTLFLAG_RW, &doingcache, 0, | SYSCTL_INT(_debug, OID_AUTO, vfscache, CTLFLAG_RW, &doingcache, 0, | ||||
"VFS namecache enabled"); | "VFS namecache enabled"); | ||||
#endif | #endif | ||||
static bool __read_mostly cache_try_smr = true; | |||||
SYSCTL_BOOL(_debug, OID_AUTO, cache_try_smr, CTLFLAG_RW, &cache_try_smr, 0, ""); | |||||
/* Export size information to userland */ | /* Export size information to userland */ | ||||
SYSCTL_INT(_debug_sizeof, OID_AUTO, namecache, CTLFLAG_RD, SYSCTL_NULL_INT_PTR, | SYSCTL_INT(_debug_sizeof, OID_AUTO, namecache, CTLFLAG_RD, SYSCTL_NULL_INT_PTR, | ||||
sizeof(struct namecache), "sizeof(struct namecache)"); | sizeof(struct namecache), "sizeof(struct namecache)"); | ||||
/* | /* | ||||
* The new name cache statistics | * The new name cache statistics | ||||
*/ | */ | ||||
static SYSCTL_NODE(_vfs, OID_AUTO, cache, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, | static SYSCTL_NODE(_vfs, OID_AUTO, cache, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, | ||||
▲ Show 20 Lines • Show All 254 Lines • ▼ Show 20 Lines | retry: | ||||
cache_lock_all_buckets(); | cache_lock_all_buckets(); | ||||
if (n_nchash != nchash + 1) { | if (n_nchash != nchash + 1) { | ||||
cache_unlock_all_buckets(); | cache_unlock_all_buckets(); | ||||
free(cntbuf, M_TEMP); | free(cntbuf, M_TEMP); | ||||
goto retry; | goto retry; | ||||
} | } | ||||
/* Scan hash tables counting entries */ | /* Scan hash tables counting entries */ | ||||
for (ncpp = nchashtbl, i = 0; i < n_nchash; ncpp++, i++) | for (ncpp = nchashtbl, i = 0; i < n_nchash; ncpp++, i++) | ||||
LIST_FOREACH(ncp, ncpp, nc_hash) | CK_LIST_FOREACH(ncp, ncpp, nc_hash) | ||||
cntbuf[i]++; | cntbuf[i]++; | ||||
cache_unlock_all_buckets(); | cache_unlock_all_buckets(); | ||||
for (error = 0, i = 0; i < n_nchash; i++) | for (error = 0, i = 0; i < n_nchash; i++) | ||||
if ((error = SYSCTL_OUT(req, &cntbuf[i], sizeof(int))) != 0) | if ((error = SYSCTL_OUT(req, &cntbuf[i], sizeof(int))) != 0) | ||||
break; | break; | ||||
free(cntbuf, M_TEMP); | free(cntbuf, M_TEMP); | ||||
return (error); | return (error); | ||||
} | } | ||||
Show All 16 Lines | sysctl_debug_hashstat_nchash(SYSCTL_HANDLER_ARGS) | ||||
cache_lock_all_buckets(); | cache_lock_all_buckets(); | ||||
n_nchash = nchash + 1; /* nchash is max index, not count */ | n_nchash = nchash + 1; /* nchash is max index, not count */ | ||||
used = 0; | used = 0; | ||||
maxlength = 0; | maxlength = 0; | ||||
/* Scan hash tables for applicable entries */ | /* Scan hash tables for applicable entries */ | ||||
for (ncpp = nchashtbl; n_nchash > 0; n_nchash--, ncpp++) { | for (ncpp = nchashtbl; n_nchash > 0; n_nchash--, ncpp++) { | ||||
count = 0; | count = 0; | ||||
LIST_FOREACH(ncp, ncpp, nc_hash) { | CK_LIST_FOREACH(ncp, ncpp, nc_hash) { | ||||
count++; | count++; | ||||
} | } | ||||
if (count) | if (count) | ||||
used++; | used++; | ||||
if (maxlength < count) | if (maxlength < count) | ||||
maxlength = count; | maxlength = count; | ||||
} | } | ||||
n_nchash = nchash + 1; | n_nchash = nchash + 1; | ||||
▲ Show 20 Lines • Show All 204 Lines • ▼ Show 20 Lines | cache_zap_locked(struct namecache *ncp, bool neg_locked) | ||||
if (!(ncp->nc_flag & NCF_NEGATIVE)) | if (!(ncp->nc_flag & NCF_NEGATIVE)) | ||||
cache_assert_vnode_locked(ncp->nc_vp); | cache_assert_vnode_locked(ncp->nc_vp); | ||||
cache_assert_vnode_locked(ncp->nc_dvp); | cache_assert_vnode_locked(ncp->nc_dvp); | ||||
cache_assert_bucket_locked(ncp, RA_WLOCKED); | cache_assert_bucket_locked(ncp, RA_WLOCKED); | ||||
CTR2(KTR_VFS, "cache_zap(%p) vp %p", ncp, | CTR2(KTR_VFS, "cache_zap(%p) vp %p", ncp, | ||||
(ncp->nc_flag & NCF_NEGATIVE) ? NULL : ncp->nc_vp); | (ncp->nc_flag & NCF_NEGATIVE) ? NULL : ncp->nc_vp); | ||||
LIST_REMOVE(ncp, nc_hash); | |||||
cache_ncp_invalidate(ncp); | |||||
CK_LIST_REMOVE(ncp, nc_hash); | |||||
if (!(ncp->nc_flag & NCF_NEGATIVE)) { | if (!(ncp->nc_flag & NCF_NEGATIVE)) { | ||||
SDT_PROBE3(vfs, namecache, zap, done, ncp->nc_dvp, | SDT_PROBE3(vfs, namecache, zap, done, ncp->nc_dvp, | ||||
ncp->nc_name, ncp->nc_vp); | ncp->nc_name, ncp->nc_vp); | ||||
TAILQ_REMOVE(&ncp->nc_vp->v_cache_dst, ncp, nc_dst); | TAILQ_REMOVE(&ncp->nc_vp->v_cache_dst, ncp, nc_dst); | ||||
if (ncp == ncp->nc_vp->v_cache_dd) | if (ncp == ncp->nc_vp->v_cache_dd) | ||||
ncp->nc_vp->v_cache_dd = NULL; | ncp->nc_vp->v_cache_dd = NULL; | ||||
} else { | } else { | ||||
SDT_PROBE2(vfs, namecache, zap_negative, done, ncp->nc_dvp, | SDT_PROBE2(vfs, namecache, zap_negative, done, ncp->nc_dvp, | ||||
▲ Show 20 Lines • Show All 136 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
struct namecache *rncp; | struct namecache *rncp; | ||||
cache_assert_bucket_locked(ncp, RA_UNLOCKED); | cache_assert_bucket_locked(ncp, RA_UNLOCKED); | ||||
cache_sort_vnodes(&dvlp, &vlp); | cache_sort_vnodes(&dvlp, &vlp); | ||||
cache_lock_vnodes(dvlp, vlp); | cache_lock_vnodes(dvlp, vlp); | ||||
rw_wlock(blp); | rw_wlock(blp); | ||||
LIST_FOREACH(rncp, (NCHHASH(hash)), nc_hash) { | CK_LIST_FOREACH(rncp, (NCHHASH(hash)), nc_hash) { | ||||
if (rncp == ncp && rncp->nc_dvp == dvp && | if (rncp == ncp && rncp->nc_dvp == dvp && | ||||
rncp->nc_nlen == cnp->cn_namelen && | rncp->nc_nlen == cnp->cn_namelen && | ||||
!bcmp(rncp->nc_name, cnp->cn_nameptr, rncp->nc_nlen)) | !bcmp(rncp->nc_name, cnp->cn_nameptr, rncp->nc_nlen)) | ||||
break; | break; | ||||
} | } | ||||
if (rncp != NULL) { | if (rncp != NULL) { | ||||
cache_zap_locked(rncp, false); | cache_zap_locked(rncp, false); | ||||
rw_wunlock(blp); | rw_wunlock(blp); | ||||
▲ Show 20 Lines • Show All 195 Lines • ▼ Show 20 Lines | if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) { | ||||
mtx_unlock(dvlp2); | mtx_unlock(dvlp2); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp); | hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp); | ||||
blp = HASH2BUCKETLOCK(hash); | blp = HASH2BUCKETLOCK(hash); | ||||
retry: | retry: | ||||
if (LIST_EMPTY(NCHHASH(hash))) | if (CK_LIST_EMPTY(NCHHASH(hash))) | ||||
goto out_no_entry; | goto out_no_entry; | ||||
rw_wlock(blp); | rw_wlock(blp); | ||||
LIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) { | CK_LIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) { | ||||
counter_u64_add(numchecks, 1); | counter_u64_add(numchecks, 1); | ||||
if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen && | if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen && | ||||
!bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen)) | !bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen)) | ||||
break; | break; | ||||
} | } | ||||
/* We failed to find an entry */ | /* We failed to find an entry */ | ||||
if (ncp == NULL) { | if (ncp == NULL) { | ||||
▲ Show 20 Lines • Show All 58 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
struct namecache_ts *ncp_ts; | struct namecache_ts *ncp_ts; | ||||
struct namecache *ncp; | struct namecache *ncp; | ||||
struct rwlock *blp; | struct rwlock *blp; | ||||
struct mtx *dvlp; | struct mtx *dvlp; | ||||
uint32_t hash; | uint32_t hash; | ||||
enum vgetstate vs; | enum vgetstate vs; | ||||
int error, ltype; | int error, ltype; | ||||
bool try_smr, doing_smr; | |||||
#ifdef DEBUG_CACHE | #ifdef DEBUG_CACHE | ||||
if (__predict_false(!doingcache)) { | if (__predict_false(!doingcache)) { | ||||
cnp->cn_flags &= ~MAKEENTRY; | cnp->cn_flags &= ~MAKEENTRY; | ||||
return (0); | return (0); | ||||
} | } | ||||
#endif | #endif | ||||
counter_u64_add(numcalls, 1); | counter_u64_add(numcalls, 1); | ||||
if (__predict_false(cnp->cn_namelen == 1 && cnp->cn_nameptr[0] == '.')) | if (__predict_false(cnp->cn_namelen == 1 && cnp->cn_nameptr[0] == '.')) | ||||
return (cache_lookup_dot(dvp, vpp, cnp, tsp, ticksp)); | return (cache_lookup_dot(dvp, vpp, cnp, tsp, ticksp)); | ||||
if ((cnp->cn_flags & MAKEENTRY) == 0) | if ((cnp->cn_flags & MAKEENTRY) == 0) | ||||
return (cache_lookup_nomakeentry(dvp, vpp, cnp, tsp, ticksp)); | return (cache_lookup_nomakeentry(dvp, vpp, cnp, tsp, ticksp)); | ||||
try_smr = atomic_load_char(&cache_try_smr); | |||||
if (cnp->cn_nameiop == CREATE) | |||||
try_smr = false; | |||||
retry: | retry: | ||||
doing_smr = false; | |||||
blp = NULL; | blp = NULL; | ||||
dvlp = NULL; | dvlp = NULL; | ||||
error = 0; | error = 0; | ||||
if (cnp->cn_namelen == 2 && | if (cnp->cn_namelen == 2 && | ||||
cnp->cn_nameptr[0] == '.' && cnp->cn_nameptr[1] == '.') { | cnp->cn_nameptr[0] == '.' && cnp->cn_nameptr[1] == '.') { | ||||
counter_u64_add(dotdothits, 1); | counter_u64_add(dotdothits, 1); | ||||
dvlp = VP2VNODELOCK(dvp); | dvlp = VP2VNODELOCK(dvp); | ||||
mtx_lock(dvlp); | mtx_lock(dvlp); | ||||
Show All 23 Lines | if ((ncp->nc_flag & (NCF_ISDOTDOT | NCF_DTS)) == | ||||
NCF_DTS && tsp != NULL) { | NCF_DTS && tsp != NULL) { | ||||
ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc); | ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc); | ||||
*tsp = ncp_ts->nc_dotdottime; | *tsp = ncp_ts->nc_dotdottime; | ||||
} | } | ||||
goto success; | goto success; | ||||
} | } | ||||
hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp); | hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp); | ||||
retry_hashed: | |||||
if (try_smr) { | |||||
vfs_smr_enter(); | |||||
doing_smr = true; | |||||
try_smr = false; | |||||
} else { | |||||
blp = HASH2BUCKETLOCK(hash); | blp = HASH2BUCKETLOCK(hash); | ||||
rw_rlock(blp); | rw_rlock(blp); | ||||
} | |||||
LIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) { | CK_LIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) { | ||||
counter_u64_add(numchecks, 1); | counter_u64_add(numchecks, 1); | ||||
if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen && | if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen && | ||||
!bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen)) | !bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen)) | ||||
break; | break; | ||||
} | } | ||||
/* We failed to find an entry */ | /* We failed to find an entry */ | ||||
if (__predict_false(ncp == NULL)) { | if (__predict_false(ncp == NULL)) { | ||||
if (doing_smr) | |||||
vfs_smr_exit(); | |||||
else | |||||
rw_runlock(blp); | rw_runlock(blp); | ||||
SDT_PROBE3(vfs, namecache, lookup, miss, dvp, cnp->cn_nameptr, | SDT_PROBE3(vfs, namecache, lookup, miss, dvp, cnp->cn_nameptr, | ||||
NULL); | NULL); | ||||
counter_u64_add(nummiss, 1); | counter_u64_add(nummiss, 1); | ||||
return (0); | return (0); | ||||
} | } | ||||
if (ncp->nc_flag & NCF_NEGATIVE) | if (ncp->nc_flag & NCF_NEGATIVE) | ||||
goto negative_success; | goto negative_success; | ||||
Show All 12 Lines | success: | ||||
* protocol. | * protocol. | ||||
*/ | */ | ||||
MPASS(dvp != *vpp); | MPASS(dvp != *vpp); | ||||
ltype = 0; /* silence gcc warning */ | ltype = 0; /* silence gcc warning */ | ||||
if (cnp->cn_flags & ISDOTDOT) { | if (cnp->cn_flags & ISDOTDOT) { | ||||
ltype = VOP_ISLOCKED(dvp); | ltype = VOP_ISLOCKED(dvp); | ||||
VOP_UNLOCK(dvp); | VOP_UNLOCK(dvp); | ||||
} | } | ||||
if (doing_smr) { | |||||
if (cache_ncp_invalid(ncp)) { | |||||
vfs_smr_exit(); | |||||
*vpp = NULL; | |||||
goto retry; | |||||
} | |||||
vs = vget_prep_smr(*vpp); | |||||
vfs_smr_exit(); | |||||
if (vs == VGET_NONE) { | |||||
*vpp = NULL; | |||||
goto retry; | |||||
} | |||||
} else { | |||||
vs = vget_prep(*vpp); | vs = vget_prep(*vpp); | ||||
cache_lookup_unlock(blp, dvlp); | cache_lookup_unlock(blp, dvlp); | ||||
} | |||||
error = vget_finish(*vpp, cnp->cn_lkflags, vs); | error = vget_finish(*vpp, cnp->cn_lkflags, vs); | ||||
if (cnp->cn_flags & ISDOTDOT) { | if (cnp->cn_flags & ISDOTDOT) { | ||||
vn_lock(dvp, ltype | LK_RETRY); | vn_lock(dvp, ltype | LK_RETRY); | ||||
if (VN_IS_DOOMED(dvp)) { | if (VN_IS_DOOMED(dvp)) { | ||||
if (error == 0) | if (error == 0) | ||||
vput(*vpp); | vput(*vpp); | ||||
*vpp = NULL; | *vpp = NULL; | ||||
return (ENOENT); | return (ENOENT); | ||||
} | } | ||||
} | } | ||||
if (error) { | if (error) { | ||||
*vpp = NULL; | *vpp = NULL; | ||||
goto retry; | goto retry; | ||||
} | } | ||||
if ((cnp->cn_flags & ISLASTCN) && | if ((cnp->cn_flags & ISLASTCN) && | ||||
(cnp->cn_lkflags & LK_TYPE_MASK) == LK_EXCLUSIVE) { | (cnp->cn_lkflags & LK_TYPE_MASK) == LK_EXCLUSIVE) { | ||||
ASSERT_VOP_ELOCKED(*vpp, "cache_lookup"); | ASSERT_VOP_ELOCKED(*vpp, "cache_lookup"); | ||||
} | } | ||||
return (-1); | return (-1); | ||||
negative_success: | negative_success: | ||||
/* We found a negative match, and want to create it, so purge */ | /* We found a negative match, and want to create it, so purge */ | ||||
if (cnp->cn_nameiop == CREATE) { | if (cnp->cn_nameiop == CREATE) { | ||||
MPASS(!doing_smr); | |||||
counter_u64_add(numnegzaps, 1); | counter_u64_add(numnegzaps, 1); | ||||
goto zap_and_exit; | goto zap_and_exit; | ||||
} | } | ||||
counter_u64_add(numneghits, 1); | if (doing_smr) { | ||||
if ((ncp->nc_flag & NCF_HOTNEGATIVE) == 0) { | |||||
/* | |||||
* We need to take locks to promote the entry. | |||||
*/ | |||||
vfs_smr_exit(); | |||||
doing_smr = false; | |||||
goto retry_hashed; | |||||
} | |||||
} else { | |||||
cache_negative_hit(ncp); | cache_negative_hit(ncp); | ||||
} | |||||
counter_u64_add(numneghits, 1); | |||||
if (ncp->nc_flag & NCF_WHITE) | if (ncp->nc_flag & NCF_WHITE) | ||||
cnp->cn_flags |= ISWHITEOUT; | cnp->cn_flags |= ISWHITEOUT; | ||||
SDT_PROBE2(vfs, namecache, lookup, hit__negative, dvp, | SDT_PROBE2(vfs, namecache, lookup, hit__negative, dvp, | ||||
ncp->nc_name); | ncp->nc_name); | ||||
cache_out_ts(ncp, tsp, ticksp); | cache_out_ts(ncp, tsp, ticksp); | ||||
if (doing_smr) | |||||
vfs_smr_exit(); | |||||
else | |||||
cache_lookup_unlock(blp, dvlp); | cache_lookup_unlock(blp, dvlp); | ||||
return (ENOENT); | return (ENOENT); | ||||
zap_and_exit: | zap_and_exit: | ||||
MPASS(!doing_smr); | |||||
if (blp != NULL) | if (blp != NULL) | ||||
error = cache_zap_rlocked_bucket(ncp, cnp, hash, blp); | error = cache_zap_rlocked_bucket(ncp, cnp, hash, blp); | ||||
else | else | ||||
error = cache_zap_locked_vnode(ncp, dvp); | error = cache_zap_locked_vnode(ncp, dvp); | ||||
if (__predict_false(error != 0)) { | if (__predict_false(error != 0)) { | ||||
zap_and_exit_bucket_fail2++; | zap_and_exit_bucket_fail2++; | ||||
cache_maybe_yield(); | cache_maybe_yield(); | ||||
goto retry; | goto retry; | ||||
▲ Show 20 Lines • Show All 317 Lines • ▼ Show 20 Lines | #endif | ||||
cache_enter_lock(&cel, dvp, vp, hash); | cache_enter_lock(&cel, dvp, vp, hash); | ||||
/* | /* | ||||
* See if this vnode or negative entry is already in the cache | * See if this vnode or negative entry is already in the cache | ||||
* with this name. This can happen with concurrent lookups of | * with this name. This can happen with concurrent lookups of | ||||
* the same path name. | * the same path name. | ||||
*/ | */ | ||||
ncpp = NCHHASH(hash); | ncpp = NCHHASH(hash); | ||||
LIST_FOREACH(n2, ncpp, nc_hash) { | CK_LIST_FOREACH(n2, ncpp, nc_hash) { | ||||
if (n2->nc_dvp == dvp && | if (n2->nc_dvp == dvp && | ||||
n2->nc_nlen == cnp->cn_namelen && | n2->nc_nlen == cnp->cn_namelen && | ||||
!bcmp(n2->nc_name, cnp->cn_nameptr, n2->nc_nlen)) { | !bcmp(n2->nc_name, cnp->cn_nameptr, n2->nc_nlen)) { | ||||
if (tsp != NULL) { | if (tsp != NULL) { | ||||
KASSERT((n2->nc_flag & NCF_TS) != 0, | KASSERT((n2->nc_flag & NCF_TS) != 0, | ||||
("no NCF_TS")); | ("no NCF_TS")); | ||||
n2_ts = __containerof(n2, struct namecache_ts, nc_nc); | n2_ts = __containerof(n2, struct namecache_ts, nc_nc); | ||||
n2_ts->nc_time = ncp_ts->nc_time; | n2_ts->nc_time = ncp_ts->nc_time; | ||||
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines | if (flag != NCF_ISDOTDOT) { | ||||
if (LIST_EMPTY(&dvp->v_cache_src)) { | if (LIST_EMPTY(&dvp->v_cache_src)) { | ||||
vhold(dvp); | vhold(dvp); | ||||
counter_u64_add(numcachehv, 1); | counter_u64_add(numcachehv, 1); | ||||
} | } | ||||
LIST_INSERT_HEAD(&dvp->v_cache_src, ncp, nc_src); | LIST_INSERT_HEAD(&dvp->v_cache_src, ncp, nc_src); | ||||
} | } | ||||
/* | /* | ||||
* Insert the new namecache entry into the appropriate chain | |||||
* within the cache entries table. | |||||
*/ | |||||
LIST_INSERT_HEAD(ncpp, ncp, nc_hash); | |||||
/* | |||||
* If the entry is "negative", we place it into the | * If the entry is "negative", we place it into the | ||||
* "negative" cache queue, otherwise, we place it into the | * "negative" cache queue, otherwise, we place it into the | ||||
* destination vnode's cache entries queue. | * destination vnode's cache entries queue. | ||||
*/ | */ | ||||
if (vp != NULL) { | if (vp != NULL) { | ||||
TAILQ_INSERT_HEAD(&vp->v_cache_dst, ncp, nc_dst); | TAILQ_INSERT_HEAD(&vp->v_cache_dst, ncp, nc_dst); | ||||
SDT_PROBE3(vfs, namecache, enter, done, dvp, ncp->nc_name, | SDT_PROBE3(vfs, namecache, enter, done, dvp, ncp->nc_name, | ||||
vp); | vp); | ||||
} else { | } else { | ||||
if (cnp->cn_flags & ISWHITEOUT) | if (cnp->cn_flags & ISWHITEOUT) | ||||
ncp->nc_flag |= NCF_WHITE; | ncp->nc_flag |= NCF_WHITE; | ||||
cache_negative_insert(ncp, false); | cache_negative_insert(ncp, false); | ||||
SDT_PROBE2(vfs, namecache, enter_negative, done, dvp, | SDT_PROBE2(vfs, namecache, enter_negative, done, dvp, | ||||
ncp->nc_name); | ncp->nc_name); | ||||
} | } | ||||
atomic_thread_fence_rel(); | |||||
/* | |||||
* Insert the new namecache entry into the appropriate chain | |||||
* within the cache entries table. | |||||
*/ | |||||
CK_LIST_INSERT_HEAD(ncpp, ncp, nc_hash); | |||||
cache_enter_unlock(&cel); | cache_enter_unlock(&cel); | ||||
if (numneg * ncnegfactor > lnumcache) | if (numneg * ncnegfactor > lnumcache) | ||||
cache_negative_zap_one(); | cache_negative_zap_one(); | ||||
cache_free(ndd); | cache_free(ndd); | ||||
return; | return; | ||||
out_unlock_free: | out_unlock_free: | ||||
cache_enter_unlock(&cel); | cache_enter_unlock(&cel); | ||||
cache_free(ncp); | cache_free(ncp); | ||||
Show All 31 Lines | cache_zone_large = uma_zcreate("L VFS Cache", | ||||
sizeof(struct namecache) + NAME_MAX + 1, | sizeof(struct namecache) + NAME_MAX + 1, | ||||
NULL, NULL, NULL, NULL, UMA_ALIGNOF(struct namecache), | NULL, NULL, NULL, NULL, UMA_ALIGNOF(struct namecache), | ||||
UMA_ZONE_ZINIT); | UMA_ZONE_ZINIT); | ||||
cache_zone_large_ts = uma_zcreate("LTS VFS Cache", | cache_zone_large_ts = uma_zcreate("LTS VFS Cache", | ||||
sizeof(struct namecache_ts) + NAME_MAX + 1, | sizeof(struct namecache_ts) + NAME_MAX + 1, | ||||
NULL, NULL, NULL, NULL, UMA_ALIGNOF(struct namecache_ts), | NULL, NULL, NULL, NULL, UMA_ALIGNOF(struct namecache_ts), | ||||
UMA_ZONE_ZINIT); | UMA_ZONE_ZINIT); | ||||
VFS_SMR_ZONE_SET(cache_zone_small); | |||||
VFS_SMR_ZONE_SET(cache_zone_small_ts); | |||||
VFS_SMR_ZONE_SET(cache_zone_large); | |||||
VFS_SMR_ZONE_SET(cache_zone_large_ts); | |||||
ncsize = desiredvnodes * ncsizefactor; | ncsize = desiredvnodes * ncsizefactor; | ||||
nchashtbl = hashinit(desiredvnodes * 2, M_VFSCACHE, &nchash); | nchashtbl = hashinit(desiredvnodes * 2, M_VFSCACHE, &nchash); | ||||
ncbuckethash = cache_roundup_2(mp_ncpus * mp_ncpus) - 1; | ncbuckethash = cache_roundup_2(mp_ncpus * mp_ncpus) - 1; | ||||
if (ncbuckethash < 7) /* arbitrarily chosen to avoid having one lock */ | if (ncbuckethash < 7) /* arbitrarily chosen to avoid having one lock */ | ||||
ncbuckethash = 7; | ncbuckethash = 7; | ||||
if (ncbuckethash > nchash) | if (ncbuckethash > nchash) | ||||
ncbuckethash = nchash; | ncbuckethash = nchash; | ||||
bucketlocks = malloc(sizeof(*bucketlocks) * numbucketlocks, M_VFSCACHE, | bucketlocks = malloc(sizeof(*bucketlocks) * numbucketlocks, M_VFSCACHE, | ||||
▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines | cache_changesize(u_long newmaxvnodes) | ||||
*/ | */ | ||||
cache_lock_all_vnodes(); | cache_lock_all_vnodes(); | ||||
cache_lock_all_buckets(); | cache_lock_all_buckets(); | ||||
old_nchashtbl = nchashtbl; | old_nchashtbl = nchashtbl; | ||||
old_nchash = nchash; | old_nchash = nchash; | ||||
nchashtbl = new_nchashtbl; | nchashtbl = new_nchashtbl; | ||||
nchash = new_nchash; | nchash = new_nchash; | ||||
for (i = 0; i <= old_nchash; i++) { | for (i = 0; i <= old_nchash; i++) { | ||||
while ((ncp = LIST_FIRST(&old_nchashtbl[i])) != NULL) { | while ((ncp = CK_LIST_FIRST(&old_nchashtbl[i])) != NULL) { | ||||
hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen, | hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen, | ||||
ncp->nc_dvp); | ncp->nc_dvp); | ||||
LIST_REMOVE(ncp, nc_hash); | CK_LIST_REMOVE(ncp, nc_hash); | ||||
LIST_INSERT_HEAD(NCHHASH(hash), ncp, nc_hash); | CK_LIST_INSERT_HEAD(NCHHASH(hash), ncp, nc_hash); | ||||
} | } | ||||
} | } | ||||
ncsize = newncsize; | ncsize = newncsize; | ||||
cache_unlock_all_buckets(); | cache_unlock_all_buckets(); | ||||
cache_unlock_all_vnodes(); | cache_unlock_all_vnodes(); | ||||
free(old_nchashtbl, M_VFSCACHE); | free(old_nchashtbl, M_VFSCACHE); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 97 Lines • ▼ Show 20 Lines | cache_purgevfs(struct mount *mp, bool force) | ||||
n_nchash = nchash + 1; | n_nchash = nchash + 1; | ||||
vlp1 = vlp2 = NULL; | vlp1 = vlp2 = NULL; | ||||
for (i = 0; i < numbucketlocks; i++) { | for (i = 0; i < numbucketlocks; i++) { | ||||
blp = (struct rwlock *)&bucketlocks[i]; | blp = (struct rwlock *)&bucketlocks[i]; | ||||
rw_wlock(blp); | rw_wlock(blp); | ||||
for (j = i; j < n_nchash; j += numbucketlocks) { | for (j = i; j < n_nchash; j += numbucketlocks) { | ||||
retry: | retry: | ||||
bucket = &nchashtbl[j]; | bucket = &nchashtbl[j]; | ||||
LIST_FOREACH_SAFE(ncp, bucket, nc_hash, nnp) { | CK_LIST_FOREACH_SAFE(ncp, bucket, nc_hash, nnp) { | ||||
cache_assert_bucket_locked(ncp, RA_WLOCKED); | cache_assert_bucket_locked(ncp, RA_WLOCKED); | ||||
if (ncp->nc_dvp->v_mount != mp) | if (ncp->nc_dvp->v_mount != mp) | ||||
continue; | continue; | ||||
error = cache_zap_wlocked_bucket_kl(ncp, blp, | error = cache_zap_wlocked_bucket_kl(ncp, blp, | ||||
&vlp1, &vlp2); | &vlp1, &vlp2); | ||||
if (error != 0) | if (error != 0) | ||||
goto retry; | goto retry; | ||||
TAILQ_INSERT_HEAD(&ncps, ncp, nc_dst); | TAILQ_INSERT_HEAD(&ncps, ncp, nc_dst); | ||||
▲ Show 20 Lines • Show All 621 Lines • Show Last 20 Lines |