Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/vfs_subr.c
Show First 20 Lines • Show All 1,305 Lines • ▼ Show 20 Lines | if (done == 0 && !retried) { | ||||
TAILQ_INSERT_HEAD(&vnode_list, mvp, v_vnodelist); | TAILQ_INSERT_HEAD(&vnode_list, mvp, v_vnodelist); | ||||
retried = true; | retried = true; | ||||
goto restart; | goto restart; | ||||
} | } | ||||
return (done); | return (done); | ||||
} | } | ||||
static int max_free_per_call = 10000; | static int max_free_per_call = 10000; | ||||
static bool recycle_vnode_bufs_pages = true; | |||||
static bool recycle_vnode_nc_src = true; | |||||
SYSCTL_INT(_debug, OID_AUTO, max_vnlru_free, CTLFLAG_RW, &max_free_per_call, 0, | SYSCTL_INT(_debug, OID_AUTO, max_vnlru_free, CTLFLAG_RW, &max_free_per_call, 0, | ||||
"limit on vnode free requests per call to the vnlru_free routine (legacy)"); | "limit on vnode free requests per call to the vnlru_free routine (legacy)"); | ||||
SYSCTL_INT(_vfs_vnode_vnlru, OID_AUTO, max_free_per_call, CTLFLAG_RW, | SYSCTL_INT(_vfs_vnode_vnlru, OID_AUTO, max_free_per_call, CTLFLAG_RW, | ||||
&max_free_per_call, 0, | &max_free_per_call, 0, | ||||
"limit on vnode free requests per call to the vnlru_free routine"); | "limit on vnode free requests per call to the vnlru_free routine"); | ||||
SYSCTL_BOOL(_vfs_vnode_vnlru, OID_AUTO, recycle_bufs_pages, CTLFLAG_RW, | |||||
&recycle_vnode_bufs_pages, 0, | |||||
"enable recycling vnodes with clean buffers and clean/dirty VM pages"); | |||||
SYSCTL_BOOL(_vfs_vnode_vnlru, OID_AUTO, recycle_nc_src, CTLFLAG_RW, | |||||
&recycle_vnode_nc_src, 0, | |||||
"enable recycling vnodes acting as namecache source"); | |||||
/* | /* | ||||
* Count the hold sources on a regular file vnode. | |||||
*/ | |||||
static void | |||||
vnlru_count_hold_sources_reg(struct vnode * restrict vp, | |||||
int * restrict vn_holdcnt, | |||||
int * restrict cleanbuf_holdcnt, | |||||
int * restrict dirtybuf_holdcnt, | |||||
int * restrict vmpage_holdcnt, | |||||
int * restrict unknown_holdcnt) | |||||
{ | |||||
struct vm_object *object; | |||||
struct bufobj *bo; | |||||
VNPASS(VREG == vp->v_type, vp); | |||||
*vn_holdcnt = atomic_load_int(&vp->v_holdcnt); | |||||
bo = &vp->v_bufobj; | |||||
*cleanbuf_holdcnt = atomic_load_int(&bo->bo_clean.bv_cnt); | |||||
*dirtybuf_holdcnt = atomic_load_int(&bo->bo_dirty.bv_cnt); | |||||
object = atomic_load_ptr(&vp->v_object); | |||||
if (object != NULL && | |||||
object->type == OBJT_VNODE && | |||||
object->resident_page_count > 0) | |||||
*vmpage_holdcnt = 1; | |||||
else | |||||
*vmpage_holdcnt = 0; | |||||
*unknown_holdcnt = *vn_holdcnt - | |||||
(*cleanbuf_holdcnt + *dirtybuf_holdcnt + *vmpage_holdcnt); | |||||
} | |||||
/* | |||||
* Count the hold sources on a directory vnode. | |||||
*/ | |||||
static void | |||||
vnlru_count_hold_sources_dir(struct vnode * restrict vp, | |||||
int * restrict vn_holdcnt, | |||||
int * restrict nc_src_holdcnt, | |||||
int * restrict unknown_holdcnt) | |||||
{ | |||||
VNPASS(VDIR == vp->v_type, vp); | |||||
*vn_holdcnt = atomic_load_int(&vp->v_holdcnt); | |||||
if (LIST_EMPTY(&vp->v_cache_src)) | |||||
*nc_src_holdcnt = 0; | |||||
else | |||||
*nc_src_holdcnt = 1; | |||||
*unknown_holdcnt = *vn_holdcnt - *nc_src_holdcnt; | |||||
} | |||||
/* | |||||
* Attempt to recycle requested amount of free vnodes. | * Attempt to recycle requested amount of free vnodes. | ||||
*/ | */ | ||||
static int | static int | ||||
vnlru_free_impl(int count, struct vfsops *mnt_op, struct vnode *mvp, bool isvnlru) | vnlru_free_impl(int count, struct vfsops *mnt_op, struct vnode *mvp, bool isvnlru) | ||||
{ | { | ||||
struct vnode *vp; | struct vnode *vp; | ||||
struct mount *mp; | struct mount *mp; | ||||
int ocount; | int ocount, vn_holdcnt, cleanbuf_holdcnt, dirtybuf_holdcnt, vmpage_holdcnt, | ||||
bool retried; | nc_src_holdcnt, unknown_holdcnt; | ||||
bool retried, *phase2_go_toggle, phase2_go; | |||||
mtx_assert(&vnode_list_mtx, MA_OWNED); | mtx_assert(&vnode_list_mtx, MA_OWNED); | ||||
if (count > max_free_per_call) | if (count > max_free_per_call) | ||||
count = max_free_per_call; | count = max_free_per_call; | ||||
if (count == 0) { | if (count == 0) { | ||||
mtx_unlock(&vnode_list_mtx); | mtx_unlock(&vnode_list_mtx); | ||||
return (0); | return (0); | ||||
} | } | ||||
Show All 23 Lines | if (__predict_false(vp == NULL)) { | ||||
*/ | */ | ||||
TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); | TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); | ||||
TAILQ_INSERT_TAIL(&vnode_list, mvp, v_vnodelist); | TAILQ_INSERT_TAIL(&vnode_list, mvp, v_vnodelist); | ||||
mtx_unlock(&vnode_list_mtx); | mtx_unlock(&vnode_list_mtx); | ||||
break; | break; | ||||
} | } | ||||
if (__predict_false(vp->v_type == VMARKER)) | if (__predict_false(vp->v_type == VMARKER)) | ||||
continue; | continue; | ||||
if (vp->v_holdcnt > 0) | |||||
continue; | |||||
/* | /* | ||||
* Don't recycle if our vnode is from different type | * Don't recycle if our vnode is from different type | ||||
* of mount point. Note that mp is type-safe, the | * of mount point. Note that mp is type-safe, the | ||||
* check does not reach unmapped address even if | * check does not reach unmapped address even if | ||||
* vnode is reclaimed. | * vnode is reclaimed. | ||||
*/ | */ | ||||
if (mnt_op != NULL && (mp = vp->v_mount) != NULL && | if (mnt_op != NULL && (mp = vp->v_mount) != NULL && | ||||
mp->mnt_op != mnt_op) { | mp->mnt_op != mnt_op) { | ||||
continue; | continue; | ||||
} | } | ||||
if (__predict_false(vp->v_type == VBAD || vp->v_type == VNON)) { | if (vp->v_type == VBAD || __predict_false(vp->v_type == VNON)) { | ||||
continue; | continue; | ||||
} | } | ||||
vn_holdcnt = atomic_load_int(&vp->v_holdcnt); | |||||
if (vn_holdcnt > 0) { | |||||
phase2_go_toggle = NULL; | |||||
phase2_go = false; | |||||
switch (vp->v_type) { | |||||
case VREG: | |||||
phase2_go_toggle = &recycle_vnode_bufs_pages; | |||||
/* | |||||
* Count the holds by the bufs and VM pages in the object, | |||||
* and compare them to the actual hold count. | |||||
*/ | |||||
vnlru_count_hold_sources_reg(vp, | |||||
&vn_holdcnt, | |||||
&cleanbuf_holdcnt, | |||||
&dirtybuf_holdcnt, | |||||
&vmpage_holdcnt, | |||||
&unknown_holdcnt); | |||||
if ((cleanbuf_holdcnt == vn_holdcnt) && | |||||
(0 == dirtybuf_holdcnt) && (0 == vmpage_holdcnt)) { | |||||
phase2_go = true; | |||||
} else if ( | |||||
((cleanbuf_holdcnt + vmpage_holdcnt) == vn_holdcnt) && | |||||
(0 == dirtybuf_holdcnt)) { | |||||
phase2_go = true; | |||||
} | |||||
break; | |||||
case VDIR: | |||||
phase2_go_toggle = &recycle_vnode_nc_src; | |||||
/* | |||||
* Count the holds by the namecache entries from this | |||||
* vnode, and compare them to the actual hold count. | |||||
*/ | |||||
vnlru_count_hold_sources_dir(vp, | |||||
&vn_holdcnt, | |||||
&nc_src_holdcnt, | |||||
&unknown_holdcnt); | |||||
if (nc_src_holdcnt == vn_holdcnt) { | |||||
phase2_go = true; | |||||
} | |||||
break; | |||||
default: | |||||
/* | |||||
* NOP; the rest of the vnode types should not happen so | |||||
* often. | |||||
*/ | |||||
break; | |||||
} | |||||
if ((NULL == phase2_go_toggle) || | |||||
!(*phase2_go_toggle) || | |||||
!phase2_go) | |||||
continue; | |||||
} | |||||
if (!vhold_recycle_free(vp)) | if (!vhold_recycle_free(vp)) | ||||
continue; | continue; | ||||
TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); | TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist); | ||||
TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist); | TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist); | ||||
mtx_unlock(&vnode_list_mtx); | mtx_unlock(&vnode_list_mtx); | ||||
/* | /* | ||||
* FIXME: ignores the return value, meaning it may be nothing | * FIXME: ignores the return value, meaning it may be nothing | ||||
* got recycled but it claims otherwise to the caller. | * got recycled but it claims otherwise to the caller. | ||||
▲ Show 20 Lines • Show All 2,356 Lines • ▼ Show 20 Lines | |||||
* it becomes recycled. Picking up such vnodes is guarded with v_holdcnt set to | * it becomes recycled. Picking up such vnodes is guarded with v_holdcnt set to | ||||
* VHOLD_NO_SMR. | * VHOLD_NO_SMR. | ||||
* | * | ||||
* Note: the vnode may gain more references after we transition the count 0->1. | * Note: the vnode may gain more references after we transition the count 0->1. | ||||
*/ | */ | ||||
static bool | static bool | ||||
vhold_recycle_free(struct vnode *vp) | vhold_recycle_free(struct vnode *vp) | ||||
{ | { | ||||
int count; | int count, vn_holdcnt, cleanbuf_holdcnt, dirtybuf_holdcnt, vmpage_holdcnt, | ||||
nc_src_holdcnt, unknown_holdcnt; | |||||
bool *phase2_go_toggle, phase2_go; | |||||
mtx_assert(&vnode_list_mtx, MA_OWNED); | mtx_assert(&vnode_list_mtx, MA_OWNED); | ||||
count = atomic_load_int(&vp->v_holdcnt); | count = atomic_load_int(&vp->v_holdcnt); | ||||
for (;;) { | for (;;) { | ||||
if (count & VHOLD_NO_SMR) { | if (count & VHOLD_NO_SMR) { | ||||
VNASSERT((count & ~VHOLD_NO_SMR) == 0, vp, | VNASSERT((count & ~VHOLD_NO_SMR) == 0, vp, | ||||
("non-zero hold count with flags %d\n", count)); | ("non-zero hold count with flags %d\n", count)); | ||||
return (false); | return (false); | ||||
} | } | ||||
VNASSERT(count >= 0, vp, ("invalid hold count %d\n", count)); | VNASSERT(count >= 0, vp, ("invalid hold count %d\n", count)); | ||||
if (count > 0) { | if (count > 0) { | ||||
/* | |||||
* Check for the vnode holds again. Refer to the phase 2 test in | |||||
* vnlru_free_impl() for the detail. | |||||
*/ | |||||
phase2_go_toggle = NULL; | |||||
phase2_go = false; | |||||
switch (vp->v_type) { | |||||
case VREG: | |||||
phase2_go_toggle = &recycle_vnode_bufs_pages; | |||||
vnlru_count_hold_sources_reg(vp, | |||||
&vn_holdcnt, | |||||
&cleanbuf_holdcnt, | |||||
&dirtybuf_holdcnt, | |||||
&vmpage_holdcnt, | |||||
&unknown_holdcnt); | |||||
if ((cleanbuf_holdcnt == vn_holdcnt) && | |||||
(0 == vmpage_holdcnt) && (0 == dirtybuf_holdcnt)) { | |||||
phase2_go = true; | |||||
} else if ( | |||||
((cleanbuf_holdcnt + vmpage_holdcnt) == vn_holdcnt) && | |||||
(0 == dirtybuf_holdcnt)) { | |||||
phase2_go = true; | |||||
} | |||||
break; | |||||
case VDIR: | |||||
phase2_go_toggle = &recycle_vnode_nc_src; | |||||
vnlru_count_hold_sources_dir(vp, | |||||
&vn_holdcnt, | |||||
&nc_src_holdcnt, | |||||
&unknown_holdcnt); | |||||
if (nc_src_holdcnt == vn_holdcnt) { | |||||
phase2_go = true; | |||||
} | |||||
break; | |||||
default: | |||||
return (false); | return (false); | ||||
} | } | ||||
if ((NULL == phase2_go_toggle) || | |||||
!(*phase2_go_toggle) || | |||||
!phase2_go) | |||||
return (false); | |||||
} | |||||
if (atomic_fcmpset_int(&vp->v_holdcnt, &count, count + 1)) { | if (atomic_fcmpset_int(&vp->v_holdcnt, &count, count + 1)) { | ||||
if (0 == count) | |||||
vfs_freevnodes_dec(); | vfs_freevnodes_dec(); | ||||
return (true); | return (true); | ||||
} | } | ||||
} | } | ||||
} | } | ||||
static void __noinline | static void __noinline | ||||
vdbatch_process(struct vdbatch *vd) | vdbatch_process(struct vdbatch *vd) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 3,558 Lines • Show Last 20 Lines |