Changeset View
Changeset View
Standalone View
Standalone View
head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c
Show First 20 Lines • Show All 333 Lines • ▼ Show 20 Lines | |||||
*/ | */ | ||||
int arc_no_grow_shift = 5; | int arc_no_grow_shift = 5; | ||||
/* | /* | ||||
* minimum lifespan of a prefetch block in clock ticks | * minimum lifespan of a prefetch block in clock ticks | ||||
* (initialized in arc_init()) | * (initialized in arc_init()) | ||||
*/ | */ | ||||
static int arc_min_prefetch_lifespan; | static int zfs_arc_min_prefetch_ms = 1; | ||||
static int zfs_arc_min_prescient_prefetch_ms = 6; | |||||
/* | /* | ||||
* If this percent of memory is free, don't throttle. | * If this percent of memory is free, don't throttle. | ||||
*/ | */ | ||||
int arc_lotsfree_percent = 10; | int arc_lotsfree_percent = 10; | ||||
static int arc_dead; | static int arc_dead; | ||||
extern boolean_t zfs_prefetch_disable; | extern boolean_t zfs_prefetch_disable; | ||||
▲ Show 20 Lines • Show All 423 Lines • ▼ Show 20 Lines | typedef struct arc_stats { | ||||
kstat_named_t arcstat_l2_write_buffer_list_iter; | kstat_named_t arcstat_l2_write_buffer_list_iter; | ||||
kstat_named_t arcstat_l2_write_buffer_list_null_iter; | kstat_named_t arcstat_l2_write_buffer_list_null_iter; | ||||
kstat_named_t arcstat_memory_throttle_count; | kstat_named_t arcstat_memory_throttle_count; | ||||
/* Not updated directly; only synced in arc_kstat_update. */ | /* Not updated directly; only synced in arc_kstat_update. */ | ||||
kstat_named_t arcstat_meta_used; | kstat_named_t arcstat_meta_used; | ||||
kstat_named_t arcstat_meta_limit; | kstat_named_t arcstat_meta_limit; | ||||
kstat_named_t arcstat_meta_max; | kstat_named_t arcstat_meta_max; | ||||
kstat_named_t arcstat_meta_min; | kstat_named_t arcstat_meta_min; | ||||
kstat_named_t arcstat_sync_wait_for_async; | kstat_named_t arcstat_async_upgrade_sync; | ||||
kstat_named_t arcstat_demand_hit_predictive_prefetch; | kstat_named_t arcstat_demand_hit_predictive_prefetch; | ||||
kstat_named_t arcstat_demand_hit_prescient_prefetch; | |||||
} arc_stats_t; | } arc_stats_t; | ||||
static arc_stats_t arc_stats = { | static arc_stats_t arc_stats = { | ||||
{ "hits", KSTAT_DATA_UINT64 }, | { "hits", KSTAT_DATA_UINT64 }, | ||||
{ "misses", KSTAT_DATA_UINT64 }, | { "misses", KSTAT_DATA_UINT64 }, | ||||
{ "demand_data_hits", KSTAT_DATA_UINT64 }, | { "demand_data_hits", KSTAT_DATA_UINT64 }, | ||||
{ "demand_data_misses", KSTAT_DATA_UINT64 }, | { "demand_data_misses", KSTAT_DATA_UINT64 }, | ||||
{ "demand_metadata_hits", KSTAT_DATA_UINT64 }, | { "demand_metadata_hits", KSTAT_DATA_UINT64 }, | ||||
▲ Show 20 Lines • Show All 80 Lines • ▼ Show 20 Lines | static arc_stats_t arc_stats = { | ||||
{ "l2_write_buffer_bytes_scanned", KSTAT_DATA_UINT64 }, | { "l2_write_buffer_bytes_scanned", KSTAT_DATA_UINT64 }, | ||||
{ "l2_write_buffer_list_iter", KSTAT_DATA_UINT64 }, | { "l2_write_buffer_list_iter", KSTAT_DATA_UINT64 }, | ||||
{ "l2_write_buffer_list_null_iter", KSTAT_DATA_UINT64 }, | { "l2_write_buffer_list_null_iter", KSTAT_DATA_UINT64 }, | ||||
{ "memory_throttle_count", KSTAT_DATA_UINT64 }, | { "memory_throttle_count", KSTAT_DATA_UINT64 }, | ||||
{ "arc_meta_used", KSTAT_DATA_UINT64 }, | { "arc_meta_used", KSTAT_DATA_UINT64 }, | ||||
{ "arc_meta_limit", KSTAT_DATA_UINT64 }, | { "arc_meta_limit", KSTAT_DATA_UINT64 }, | ||||
{ "arc_meta_max", KSTAT_DATA_UINT64 }, | { "arc_meta_max", KSTAT_DATA_UINT64 }, | ||||
{ "arc_meta_min", KSTAT_DATA_UINT64 }, | { "arc_meta_min", KSTAT_DATA_UINT64 }, | ||||
{ "sync_wait_for_async", KSTAT_DATA_UINT64 }, | { "async_upgrade_sync", KSTAT_DATA_UINT64 }, | ||||
{ "demand_hit_predictive_prefetch", KSTAT_DATA_UINT64 }, | { "demand_hit_predictive_prefetch", KSTAT_DATA_UINT64 }, | ||||
{ "demand_hit_prescient_prefetch", KSTAT_DATA_UINT64 }, | |||||
}; | }; | ||||
#define ARCSTAT(stat) (arc_stats.stat.value.ui64) | #define ARCSTAT(stat) (arc_stats.stat.value.ui64) | ||||
#define ARCSTAT_INCR(stat, val) \ | #define ARCSTAT_INCR(stat, val) \ | ||||
atomic_add_64(&arc_stats.stat.value.ui64, (val)) | atomic_add_64(&arc_stats.stat.value.ui64, (val)) | ||||
#define ARCSTAT_BUMP(stat) ARCSTAT_INCR(stat, 1) | #define ARCSTAT_BUMP(stat) ARCSTAT_INCR(stat, 1) | ||||
▲ Show 20 Lines • Show All 79 Lines • ▼ Show 20 Lines | |||||
static int arc_no_grow; /* Don't try to grow cache size */ | static int arc_no_grow; /* Don't try to grow cache size */ | ||||
static uint64_t arc_tempreserve; | static uint64_t arc_tempreserve; | ||||
static uint64_t arc_loaned_bytes; | static uint64_t arc_loaned_bytes; | ||||
typedef struct arc_callback arc_callback_t; | typedef struct arc_callback arc_callback_t; | ||||
struct arc_callback { | struct arc_callback { | ||||
void *acb_private; | void *acb_private; | ||||
arc_done_func_t *acb_done; | arc_read_done_func_t *acb_done; | ||||
arc_buf_t *acb_buf; | arc_buf_t *acb_buf; | ||||
boolean_t acb_compressed; | boolean_t acb_compressed; | ||||
zio_t *acb_zio_dummy; | zio_t *acb_zio_dummy; | ||||
zio_t *acb_zio_head; | |||||
arc_callback_t *acb_next; | arc_callback_t *acb_next; | ||||
}; | }; | ||||
typedef struct arc_write_callback arc_write_callback_t; | typedef struct arc_write_callback arc_write_callback_t; | ||||
struct arc_write_callback { | struct arc_write_callback { | ||||
void *awcb_private; | void *awcb_private; | ||||
arc_done_func_t *awcb_ready; | arc_write_done_func_t *awcb_ready; | ||||
arc_done_func_t *awcb_children_ready; | arc_write_done_func_t *awcb_children_ready; | ||||
arc_done_func_t *awcb_physdone; | arc_write_done_func_t *awcb_physdone; | ||||
arc_done_func_t *awcb_done; | arc_write_done_func_t *awcb_done; | ||||
arc_buf_t *awcb_buf; | arc_buf_t *awcb_buf; | ||||
}; | }; | ||||
/* | /* | ||||
* ARC buffers are separated into multiple structs as a memory saving measure: | * ARC buffers are separated into multiple structs as a memory saving measure: | ||||
* - Common fields struct, always defined, and embedded within it: | * - Common fields struct, always defined, and embedded within it: | ||||
* - L2-only fields, always allocated but undefined when not in L2ARC | * - L2-only fields, always allocated but undefined when not in L2ARC | ||||
* - L1-only fields, only allocated when in L1ARC | * - L1-only fields, only allocated when in L1ARC | ||||
* | * | ||||
▲ Show 20 Lines • Show All 223 Lines • ▼ Show 20 Lines | |||||
#define GHOST_STATE(state) \ | #define GHOST_STATE(state) \ | ||||
((state) == arc_mru_ghost || (state) == arc_mfu_ghost || \ | ((state) == arc_mru_ghost || (state) == arc_mfu_ghost || \ | ||||
(state) == arc_l2c_only) | (state) == arc_l2c_only) | ||||
#define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE) | #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE) | ||||
#define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) | #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) | ||||
#define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_FLAG_IO_ERROR) | #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_FLAG_IO_ERROR) | ||||
#define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_FLAG_PREFETCH) | #define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_FLAG_PREFETCH) | ||||
#define HDR_PRESCIENT_PREFETCH(hdr) \ | |||||
((hdr)->b_flags & ARC_FLAG_PRESCIENT_PREFETCH) | |||||
#define HDR_COMPRESSION_ENABLED(hdr) \ | #define HDR_COMPRESSION_ENABLED(hdr) \ | ||||
((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC) | ((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC) | ||||
#define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_FLAG_L2CACHE) | #define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_FLAG_L2CACHE) | ||||
#define HDR_L2_READING(hdr) \ | #define HDR_L2_READING(hdr) \ | ||||
(((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) && \ | (((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) && \ | ||||
((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)) | ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)) | ||||
#define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITING) | #define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITING) | ||||
▲ Show 20 Lines • Show All 147 Lines • ▼ Show 20 Lines | SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_ghost_metadata_esize, CTLFLAG_RD, | ||||
"size of metadata in mfu ghost state"); | "size of metadata in mfu ghost state"); | ||||
SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_ghost_data_esize, CTLFLAG_RD, | SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_ghost_data_esize, CTLFLAG_RD, | ||||
&ARC_mfu_ghost.arcs_esize[ARC_BUFC_DATA].rc_count, 0, | &ARC_mfu_ghost.arcs_esize[ARC_BUFC_DATA].rc_count, 0, | ||||
"size of data in mfu ghost state"); | "size of data in mfu ghost state"); | ||||
SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2c_only_size, CTLFLAG_RD, | SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2c_only_size, CTLFLAG_RD, | ||||
&ARC_l2c_only.arcs_size.rc_count, 0, "size of mru state"); | &ARC_l2c_only.arcs_size.rc_count, 0, "size of mru state"); | ||||
SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_min_prfetch_ms, CTLFLAG_RW, | |||||
&zfs_arc_min_prefetch_ms, 0, "Min life of prefetch block in ms"); | |||||
SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_min_prescient_prefetch_ms, CTLFLAG_RW, | |||||
&zfs_arc_min_prescient_prefetch_ms, 0, "Min life oof prescient prefetched block in ms"); | |||||
/* | /* | ||||
* L2ARC Internals | * L2ARC Internals | ||||
*/ | */ | ||||
struct l2arc_dev { | struct l2arc_dev { | ||||
vdev_t *l2ad_vdev; /* vdev */ | vdev_t *l2ad_vdev; /* vdev */ | ||||
spa_t *l2ad_spa; /* spa */ | spa_t *l2ad_spa; /* spa */ | ||||
uint64_t l2ad_hand; /* next write location */ | uint64_t l2ad_hand; /* next write location */ | ||||
uint64_t l2ad_start; /* first addr on device */ | uint64_t l2ad_start; /* first addr on device */ | ||||
▲ Show 20 Lines • Show All 2,136 Lines • ▼ Show 20 Lines | |||||
* - arc_mfu_ghost -> arc_l2c_only | * - arc_mfu_ghost -> arc_l2c_only | ||||
* - arc_mfu_ghost -> deleted | * - arc_mfu_ghost -> deleted | ||||
*/ | */ | ||||
static int64_t | static int64_t | ||||
arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) | arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) | ||||
{ | { | ||||
arc_state_t *evicted_state, *state; | arc_state_t *evicted_state, *state; | ||||
int64_t bytes_evicted = 0; | int64_t bytes_evicted = 0; | ||||
int min_lifetime = HDR_PRESCIENT_PREFETCH(hdr) ? | |||||
zfs_arc_min_prescient_prefetch_ms : zfs_arc_min_prefetch_ms; | |||||
ASSERT(MUTEX_HELD(hash_lock)); | ASSERT(MUTEX_HELD(hash_lock)); | ||||
ASSERT(HDR_HAS_L1HDR(hdr)); | ASSERT(HDR_HAS_L1HDR(hdr)); | ||||
state = hdr->b_l1hdr.b_state; | state = hdr->b_l1hdr.b_state; | ||||
if (GHOST_STATE(state)) { | if (GHOST_STATE(state)) { | ||||
ASSERT(!HDR_IO_IN_PROGRESS(hdr)); | ASSERT(!HDR_IO_IN_PROGRESS(hdr)); | ||||
ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); | ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); | ||||
Show All 36 Lines | arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) | ||||
} | } | ||||
ASSERT(state == arc_mru || state == arc_mfu); | ASSERT(state == arc_mru || state == arc_mfu); | ||||
evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; | evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; | ||||
/* prefetch buffers have a minimum lifespan */ | /* prefetch buffers have a minimum lifespan */ | ||||
if (HDR_IO_IN_PROGRESS(hdr) || | if (HDR_IO_IN_PROGRESS(hdr) || | ||||
((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) && | ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) && | ||||
ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access < | ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access < min_lifetime * hz)) { | ||||
arc_min_prefetch_lifespan)) { | |||||
ARCSTAT_BUMP(arcstat_evict_skip); | ARCSTAT_BUMP(arcstat_evict_skip); | ||||
return (bytes_evicted); | return (bytes_evicted); | ||||
} | } | ||||
ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt)); | ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt)); | ||||
while (hdr->b_l1hdr.b_buf) { | while (hdr->b_l1hdr.b_buf) { | ||||
arc_buf_t *buf = hdr->b_l1hdr.b_buf; | arc_buf_t *buf = hdr->b_l1hdr.b_buf; | ||||
if (!mutex_tryenter(&buf->b_evict_lock)) { | if (!mutex_tryenter(&buf->b_evict_lock)) { | ||||
▲ Show 20 Lines • Show All 1,354 Lines • ▼ Show 20 Lines | if (hdr->b_l1hdr.b_state == arc_anon) { | ||||
/* | /* | ||||
* If this buffer is here because of a prefetch, then either: | * If this buffer is here because of a prefetch, then either: | ||||
* - clear the flag if this is a "referencing" read | * - clear the flag if this is a "referencing" read | ||||
* (any subsequent access will bump this into the MFU state). | * (any subsequent access will bump this into the MFU state). | ||||
* or | * or | ||||
* - move the buffer to the head of the list if this is | * - move the buffer to the head of the list if this is | ||||
* another prefetch (to make it less likely to be evicted). | * another prefetch (to make it less likely to be evicted). | ||||
*/ | */ | ||||
if (HDR_PREFETCH(hdr)) { | if (HDR_PREFETCH(hdr) || HDR_PRESCIENT_PREFETCH(hdr)) { | ||||
if (refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) { | if (refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) { | ||||
/* link protected by hash lock */ | /* link protected by hash lock */ | ||||
ASSERT(multilist_link_active( | ASSERT(multilist_link_active( | ||||
&hdr->b_l1hdr.b_arc_node)); | &hdr->b_l1hdr.b_arc_node)); | ||||
} else { | } else { | ||||
arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH); | arc_hdr_clear_flags(hdr, | ||||
ARC_FLAG_PREFETCH | | |||||
ARC_FLAG_PRESCIENT_PREFETCH); | |||||
ARCSTAT_BUMP(arcstat_mru_hits); | ARCSTAT_BUMP(arcstat_mru_hits); | ||||
} | } | ||||
hdr->b_l1hdr.b_arc_access = now; | hdr->b_l1hdr.b_arc_access = now; | ||||
return; | return; | ||||
} | } | ||||
/* | /* | ||||
* This buffer has been "accessed" only once so far, | * This buffer has been "accessed" only once so far, | ||||
Show All 14 Lines | arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) | ||||
} else if (hdr->b_l1hdr.b_state == arc_mru_ghost) { | } else if (hdr->b_l1hdr.b_state == arc_mru_ghost) { | ||||
arc_state_t *new_state; | arc_state_t *new_state; | ||||
/* | /* | ||||
* This buffer has been "accessed" recently, but | * This buffer has been "accessed" recently, but | ||||
* was evicted from the cache. Move it to the | * was evicted from the cache. Move it to the | ||||
* MFU state. | * MFU state. | ||||
*/ | */ | ||||
if (HDR_PREFETCH(hdr)) { | if (HDR_PREFETCH(hdr) || HDR_PRESCIENT_PREFETCH(hdr)) { | ||||
new_state = arc_mru; | new_state = arc_mru; | ||||
if (refcount_count(&hdr->b_l1hdr.b_refcnt) > 0) | if (refcount_count(&hdr->b_l1hdr.b_refcnt) > 0) { | ||||
arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH); | arc_hdr_clear_flags(hdr, | ||||
ARC_FLAG_PREFETCH | | |||||
ARC_FLAG_PRESCIENT_PREFETCH); | |||||
} | |||||
DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr); | DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr); | ||||
} else { | } else { | ||||
new_state = arc_mfu; | new_state = arc_mfu; | ||||
DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); | DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); | ||||
} | } | ||||
hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); | hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); | ||||
arc_change_state(new_state, hdr, hash_lock); | arc_change_state(new_state, hdr, hash_lock); | ||||
ARCSTAT_BUMP(arcstat_mru_ghost_hits); | ARCSTAT_BUMP(arcstat_mru_ghost_hits); | ||||
} else if (hdr->b_l1hdr.b_state == arc_mfu) { | } else if (hdr->b_l1hdr.b_state == arc_mfu) { | ||||
/* | /* | ||||
* This buffer has been accessed more than once and is | * This buffer has been accessed more than once and is | ||||
* still in the cache. Keep it in the MFU state. | * still in the cache. Keep it in the MFU state. | ||||
* | * | ||||
* NOTE: an add_reference() that occurred when we did | * NOTE: an add_reference() that occurred when we did | ||||
* the arc_read() will have kicked this off the list. | * the arc_read() will have kicked this off the list. | ||||
* If it was a prefetch, we will explicitly move it to | * If it was a prefetch, we will explicitly move it to | ||||
* the head of the list now. | * the head of the list now. | ||||
*/ | */ | ||||
if ((HDR_PREFETCH(hdr)) != 0) { | |||||
ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); | |||||
/* link protected by hash_lock */ | |||||
ASSERT(multilist_link_active(&hdr->b_l1hdr.b_arc_node)); | |||||
} | |||||
ARCSTAT_BUMP(arcstat_mfu_hits); | ARCSTAT_BUMP(arcstat_mfu_hits); | ||||
hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); | hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); | ||||
} else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) { | } else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) { | ||||
arc_state_t *new_state = arc_mfu; | arc_state_t *new_state = arc_mfu; | ||||
/* | /* | ||||
* This buffer has been accessed more than once but has | * This buffer has been accessed more than once but has | ||||
* been evicted from the cache. Move it back to the | * been evicted from the cache. Move it back to the | ||||
* MFU state. | * MFU state. | ||||
*/ | */ | ||||
if (HDR_PREFETCH(hdr)) { | if (HDR_PREFETCH(hdr) || HDR_PRESCIENT_PREFETCH(hdr)) { | ||||
/* | /* | ||||
* This is a prefetch access... | * This is a prefetch access... | ||||
* move this block back to the MRU state. | * move this block back to the MRU state. | ||||
*/ | */ | ||||
ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt)); | |||||
new_state = arc_mru; | new_state = arc_mru; | ||||
} | } | ||||
hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); | hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); | ||||
DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); | DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); | ||||
arc_change_state(new_state, hdr, hash_lock); | arc_change_state(new_state, hdr, hash_lock); | ||||
ARCSTAT_BUMP(arcstat_mfu_ghost_hits); | ARCSTAT_BUMP(arcstat_mfu_ghost_hits); | ||||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | arc_buf_access(arc_buf_t *buf) | ||||
arc_access(hdr, hash_lock); | arc_access(hdr, hash_lock); | ||||
mutex_exit(hash_lock); | mutex_exit(hash_lock); | ||||
ARCSTAT_BUMP(arcstat_hits); | ARCSTAT_BUMP(arcstat_hits); | ||||
ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr), | ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr), | ||||
demand, prefetch, !HDR_ISTYPE_METADATA(hdr), data, metadata, hits); | demand, prefetch, !HDR_ISTYPE_METADATA(hdr), data, metadata, hits); | ||||
} | } | ||||
/* a generic arc_done_func_t which you can use */ | /* a generic arc_read_done_func_t which you can use */ | ||||
/* ARGSUSED */ | /* ARGSUSED */ | ||||
void | void | ||||
arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg) | arc_bcopy_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp, | ||||
arc_buf_t *buf, void *arg) | |||||
{ | { | ||||
if (zio == NULL || zio->io_error == 0) | if (buf == NULL) | ||||
return; | |||||
bcopy(buf->b_data, arg, arc_buf_size(buf)); | bcopy(buf->b_data, arg, arc_buf_size(buf)); | ||||
arc_buf_destroy(buf, arg); | arc_buf_destroy(buf, arg); | ||||
} | } | ||||
/* a generic arc_done_func_t */ | /* a generic arc_read_done_func_t */ | ||||
/* ARGSUSED */ | |||||
void | void | ||||
arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg) | arc_getbuf_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp, | ||||
arc_buf_t *buf, void *arg) | |||||
{ | { | ||||
arc_buf_t **bufp = arg; | arc_buf_t **bufp = arg; | ||||
if (zio && zio->io_error) { | |||||
arc_buf_destroy(buf, arg); | if (buf == NULL) { | ||||
*bufp = NULL; | *bufp = NULL; | ||||
} else { | } else { | ||||
*bufp = buf; | *bufp = buf; | ||||
ASSERT(buf->b_data); | ASSERT(buf->b_data); | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
Show All 15 Lines | |||||
static void | static void | ||||
arc_read_done(zio_t *zio) | arc_read_done(zio_t *zio) | ||||
{ | { | ||||
arc_buf_hdr_t *hdr = zio->io_private; | arc_buf_hdr_t *hdr = zio->io_private; | ||||
kmutex_t *hash_lock = NULL; | kmutex_t *hash_lock = NULL; | ||||
arc_callback_t *callback_list; | arc_callback_t *callback_list; | ||||
arc_callback_t *acb; | arc_callback_t *acb; | ||||
boolean_t freeable = B_FALSE; | boolean_t freeable = B_FALSE; | ||||
boolean_t no_zio_error = (zio->io_error == 0); | |||||
/* | /* | ||||
* The hdr was inserted into hash-table and removed from lists | * The hdr was inserted into hash-table and removed from lists | ||||
* prior to starting I/O. We should find this header, since | * prior to starting I/O. We should find this header, since | ||||
* it's in the hash table, and it should be legit since it's | * it's in the hash table, and it should be legit since it's | ||||
* not possible to evict it during the I/O. The only possible | * not possible to evict it during the I/O. The only possible | ||||
* reason for it not to be found is if we were freed during the | * reason for it not to be found is if we were freed during the | ||||
* read. | * read. | ||||
Show All 9 Lines | arc_buf_hdr_t *found = buf_hash_find(hdr->b_spa, zio->io_bp, | ||||
&hash_lock); | &hash_lock); | ||||
ASSERT((found == hdr && | ASSERT((found == hdr && | ||||
DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) || | DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) || | ||||
(found == hdr && HDR_L2_READING(hdr))); | (found == hdr && HDR_L2_READING(hdr))); | ||||
ASSERT3P(hash_lock, !=, NULL); | ASSERT3P(hash_lock, !=, NULL); | ||||
} | } | ||||
if (no_zio_error) { | if (zio->io_error == 0) { | ||||
/* byteswap if necessary */ | /* byteswap if necessary */ | ||||
if (BP_SHOULD_BYTESWAP(zio->io_bp)) { | if (BP_SHOULD_BYTESWAP(zio->io_bp)) { | ||||
if (BP_GET_LEVEL(zio->io_bp) > 0) { | if (BP_GET_LEVEL(zio->io_bp) > 0) { | ||||
hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64; | hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64; | ||||
} else { | } else { | ||||
hdr->b_l1hdr.b_byteswap = | hdr->b_l1hdr.b_byteswap = | ||||
DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp)); | DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp)); | ||||
} | } | ||||
} else { | } else { | ||||
hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; | hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; | ||||
} | } | ||||
} | } | ||||
arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED); | arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED); | ||||
if (l2arc_noprefetch && HDR_PREFETCH(hdr)) | if (l2arc_noprefetch && HDR_PREFETCH(hdr)) | ||||
arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE); | arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE); | ||||
callback_list = hdr->b_l1hdr.b_acb; | callback_list = hdr->b_l1hdr.b_acb; | ||||
ASSERT3P(callback_list, !=, NULL); | ASSERT3P(callback_list, !=, NULL); | ||||
if (hash_lock && no_zio_error && hdr->b_l1hdr.b_state == arc_anon) { | if (hash_lock && zio->io_error == 0 && | ||||
hdr->b_l1hdr.b_state == arc_anon) { | |||||
/* | /* | ||||
* Only call arc_access on anonymous buffers. This is because | * Only call arc_access on anonymous buffers. This is because | ||||
* if we've issued an I/O for an evicted buffer, we've already | * if we've issued an I/O for an evicted buffer, we've already | ||||
* called arc_access (to prevent any simultaneous readers from | * called arc_access (to prevent any simultaneous readers from | ||||
* getting confused). | * getting confused). | ||||
*/ | */ | ||||
arc_access(hdr, hash_lock); | arc_access(hdr, hash_lock); | ||||
} | } | ||||
/* | /* | ||||
* If a read request has a callback (i.e. acb_done is not NULL), then we | * If a read request has a callback (i.e. acb_done is not NULL), then we | ||||
* make a buf containing the data according to the parameters which were | * make a buf containing the data according to the parameters which were | ||||
* passed in. The implementation of arc_buf_alloc_impl() ensures that we | * passed in. The implementation of arc_buf_alloc_impl() ensures that we | ||||
* aren't needlessly decompressing the data multiple times. | * aren't needlessly decompressing the data multiple times. | ||||
*/ | */ | ||||
int callback_cnt = 0; | int callback_cnt = 0; | ||||
for (acb = callback_list; acb != NULL; acb = acb->acb_next) { | for (acb = callback_list; acb != NULL; acb = acb->acb_next) { | ||||
if (!acb->acb_done) | if (!acb->acb_done) | ||||
continue; | continue; | ||||
/* This is a demand read since prefetches don't use callbacks */ | |||||
callback_cnt++; | callback_cnt++; | ||||
if (zio->io_error != 0) | |||||
continue; | |||||
int error = arc_buf_alloc_impl(hdr, acb->acb_private, | int error = arc_buf_alloc_impl(hdr, acb->acb_private, | ||||
acb->acb_compressed, no_zio_error, &acb->acb_buf); | acb->acb_compressed, | ||||
if (no_zio_error) { | B_TRUE, &acb->acb_buf); | ||||
if (error != 0) { | |||||
arc_buf_destroy(acb->acb_buf, acb->acb_private); | |||||
acb->acb_buf = NULL; | |||||
} | |||||
if (zio->io_error == 0) | |||||
zio->io_error = error; | zio->io_error = error; | ||||
} | } | ||||
} | |||||
hdr->b_l1hdr.b_acb = NULL; | hdr->b_l1hdr.b_acb = NULL; | ||||
arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); | arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); | ||||
if (callback_cnt == 0) { | if (callback_cnt == 0) { | ||||
ASSERT(HDR_PREFETCH(hdr)); | ASSERT(HDR_PREFETCH(hdr)); | ||||
ASSERT0(hdr->b_l1hdr.b_bufcnt); | ASSERT0(hdr->b_l1hdr.b_bufcnt); | ||||
ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); | ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); | ||||
} | } | ||||
ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) || | ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) || | ||||
callback_list != NULL); | callback_list != NULL); | ||||
if (no_zio_error) { | if (zio->io_error == 0) { | ||||
arc_hdr_verify(hdr, zio->io_bp); | arc_hdr_verify(hdr, zio->io_bp); | ||||
} else { | } else { | ||||
arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR); | arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR); | ||||
if (hdr->b_l1hdr.b_state != arc_anon) | if (hdr->b_l1hdr.b_state != arc_anon) | ||||
arc_change_state(arc_anon, hdr, hash_lock); | arc_change_state(arc_anon, hdr, hash_lock); | ||||
if (HDR_IN_HASH_TABLE(hdr)) | if (HDR_IN_HASH_TABLE(hdr)) | ||||
buf_hash_remove(hdr); | buf_hash_remove(hdr); | ||||
freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt); | freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt); | ||||
Show All 16 Lines | if (hash_lock != NULL) { | ||||
* in the cache). | * in the cache). | ||||
*/ | */ | ||||
ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); | ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); | ||||
freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt); | freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt); | ||||
} | } | ||||
/* execute each callback and free its structure */ | /* execute each callback and free its structure */ | ||||
while ((acb = callback_list) != NULL) { | while ((acb = callback_list) != NULL) { | ||||
if (acb->acb_done) | if (acb->acb_done) { | ||||
acb->acb_done(zio, acb->acb_buf, acb->acb_private); | acb->acb_done(zio, &zio->io_bookmark, zio->io_bp, | ||||
acb->acb_buf, acb->acb_private); | |||||
} | |||||
if (acb->acb_zio_dummy != NULL) { | if (acb->acb_zio_dummy != NULL) { | ||||
acb->acb_zio_dummy->io_error = zio->io_error; | acb->acb_zio_dummy->io_error = zio->io_error; | ||||
zio_nowait(acb->acb_zio_dummy); | zio_nowait(acb->acb_zio_dummy); | ||||
} | } | ||||
callback_list = acb->acb_next; | callback_list = acb->acb_next; | ||||
kmem_free(acb, sizeof (arc_callback_t)); | kmem_free(acb, sizeof (arc_callback_t)); | ||||
Show All 17 Lines | |||||
* results); or, if this is a read with a "done" func, add a record | * results); or, if this is a read with a "done" func, add a record | ||||
* to the read to invoke the "done" func when the read completes, | * to the read to invoke the "done" func when the read completes, | ||||
* and return; or just return. | * and return; or just return. | ||||
* | * | ||||
* arc_read_done() will invoke all the requested "done" functions | * arc_read_done() will invoke all the requested "done" functions | ||||
* for readers of this block. | * for readers of this block. | ||||
*/ | */ | ||||
int | int | ||||
arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_done_func_t *done, | arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_read_done_func_t *done, | ||||
void *private, zio_priority_t priority, int zio_flags, | void *private, zio_priority_t priority, int zio_flags, | ||||
arc_flags_t *arc_flags, const zbookmark_phys_t *zb) | arc_flags_t *arc_flags, const zbookmark_phys_t *zb) | ||||
{ | { | ||||
arc_buf_hdr_t *hdr = NULL; | arc_buf_hdr_t *hdr = NULL; | ||||
kmutex_t *hash_lock = NULL; | kmutex_t *hash_lock = NULL; | ||||
zio_t *rzio; | zio_t *rzio; | ||||
uint64_t guid = spa_load_guid(spa); | uint64_t guid = spa_load_guid(spa); | ||||
boolean_t compressed_read = (zio_flags & ZIO_FLAG_RAW) != 0; | boolean_t compressed_read = (zio_flags & ZIO_FLAG_RAW) != 0; | ||||
int rc = 0; | |||||
ASSERT(!BP_IS_EMBEDDED(bp) || | ASSERT(!BP_IS_EMBEDDED(bp) || | ||||
BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA); | BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA); | ||||
top: | top: | ||||
if (!BP_IS_EMBEDDED(bp)) { | if (!BP_IS_EMBEDDED(bp)) { | ||||
/* | /* | ||||
* Embedded BP's have no DVA and require no I/O to "read". | * Embedded BP's have no DVA and require no I/O to "read". | ||||
* Create an anonymous arc buf to back it. | * Create an anonymous arc buf to back it. | ||||
*/ | */ | ||||
hdr = buf_hash_find(guid, bp, &hash_lock); | hdr = buf_hash_find(guid, bp, &hash_lock); | ||||
} | } | ||||
if (hdr != NULL && HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_pabd != NULL) { | if (hdr != NULL && HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_pabd != NULL) { | ||||
arc_buf_t *buf = NULL; | arc_buf_t *buf = NULL; | ||||
*arc_flags |= ARC_FLAG_CACHED; | *arc_flags |= ARC_FLAG_CACHED; | ||||
if (HDR_IO_IN_PROGRESS(hdr)) { | if (HDR_IO_IN_PROGRESS(hdr)) { | ||||
zio_t *head_zio = hdr->b_l1hdr.b_acb->acb_zio_head; | |||||
ASSERT3P(head_zio, !=, NULL); | |||||
if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) && | if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) && | ||||
priority == ZIO_PRIORITY_SYNC_READ) { | priority == ZIO_PRIORITY_SYNC_READ) { | ||||
/* | /* | ||||
* This sync read must wait for an | * This is a sync read that needs to wait for | ||||
* in-progress async read (e.g. a predictive | * an in-flight async read. Request that the | ||||
* prefetch). Async reads are queued | * zio have its priority upgraded. | ||||
* separately at the vdev_queue layer, so | |||||
* this is a form of priority inversion. | |||||
* Ideally, we would "inherit" the demand | |||||
* i/o's priority by moving the i/o from | |||||
* the async queue to the synchronous queue, | |||||
* but there is currently no mechanism to do | |||||
* so. Track this so that we can evaluate | |||||
* the magnitude of this potential performance | |||||
* problem. | |||||
* | |||||
* Note that if the prefetch i/o is already | |||||
* active (has been issued to the device), | |||||
* the prefetch improved performance, because | |||||
* we issued it sooner than we would have | |||||
* without the prefetch. | |||||
*/ | */ | ||||
DTRACE_PROBE1(arc__sync__wait__for__async, | zio_change_priority(head_zio, priority); | ||||
DTRACE_PROBE1(arc__async__upgrade__sync, | |||||
arc_buf_hdr_t *, hdr); | arc_buf_hdr_t *, hdr); | ||||
ARCSTAT_BUMP(arcstat_sync_wait_for_async); | ARCSTAT_BUMP(arcstat_async_upgrade_sync); | ||||
} | } | ||||
if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) { | if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) { | ||||
arc_hdr_clear_flags(hdr, | arc_hdr_clear_flags(hdr, | ||||
ARC_FLAG_PREDICTIVE_PREFETCH); | ARC_FLAG_PREDICTIVE_PREFETCH); | ||||
} | } | ||||
if (*arc_flags & ARC_FLAG_WAIT) { | if (*arc_flags & ARC_FLAG_WAIT) { | ||||
cv_wait(&hdr->b_l1hdr.b_cv, hash_lock); | cv_wait(&hdr->b_l1hdr.b_cv, hash_lock); | ||||
Show All 10 Lines | if (HDR_IO_IN_PROGRESS(hdr)) { | ||||
acb->acb_done = done; | acb->acb_done = done; | ||||
acb->acb_private = private; | acb->acb_private = private; | ||||
acb->acb_compressed = compressed_read; | acb->acb_compressed = compressed_read; | ||||
if (pio != NULL) | if (pio != NULL) | ||||
acb->acb_zio_dummy = zio_null(pio, | acb->acb_zio_dummy = zio_null(pio, | ||||
spa, NULL, NULL, NULL, zio_flags); | spa, NULL, NULL, NULL, zio_flags); | ||||
ASSERT3P(acb->acb_done, !=, NULL); | ASSERT3P(acb->acb_done, !=, NULL); | ||||
acb->acb_zio_head = head_zio; | |||||
acb->acb_next = hdr->b_l1hdr.b_acb; | acb->acb_next = hdr->b_l1hdr.b_acb; | ||||
hdr->b_l1hdr.b_acb = acb; | hdr->b_l1hdr.b_acb = acb; | ||||
mutex_exit(hash_lock); | mutex_exit(hash_lock); | ||||
return (0); | return (0); | ||||
} | } | ||||
mutex_exit(hash_lock); | mutex_exit(hash_lock); | ||||
return (0); | return (0); | ||||
} | } | ||||
Show All 11 Lines | if (done) { | ||||
DTRACE_PROBE1( | DTRACE_PROBE1( | ||||
arc__demand__hit__predictive__prefetch, | arc__demand__hit__predictive__prefetch, | ||||
arc_buf_hdr_t *, hdr); | arc_buf_hdr_t *, hdr); | ||||
ARCSTAT_BUMP( | ARCSTAT_BUMP( | ||||
arcstat_demand_hit_predictive_prefetch); | arcstat_demand_hit_predictive_prefetch); | ||||
arc_hdr_clear_flags(hdr, | arc_hdr_clear_flags(hdr, | ||||
ARC_FLAG_PREDICTIVE_PREFETCH); | ARC_FLAG_PREDICTIVE_PREFETCH); | ||||
} | } | ||||
ASSERT(!BP_IS_EMBEDDED(bp) || !BP_IS_HOLE(bp)); | |||||
if (hdr->b_flags & ARC_FLAG_PRESCIENT_PREFETCH) { | |||||
ARCSTAT_BUMP( | |||||
arcstat_demand_hit_prescient_prefetch); | |||||
arc_hdr_clear_flags(hdr, | |||||
ARC_FLAG_PRESCIENT_PREFETCH); | |||||
} | |||||
ASSERT(!BP_IS_EMBEDDED(bp) || !BP_IS_HOLE(bp)); | |||||
/* Get a buf with the desired data in it. */ | /* Get a buf with the desired data in it. */ | ||||
VERIFY0(arc_buf_alloc_impl(hdr, private, | rc = arc_buf_alloc_impl(hdr, private, | ||||
compressed_read, B_TRUE, &buf)); | compressed_read, B_TRUE, &buf); | ||||
if (rc != 0) { | |||||
arc_buf_destroy(buf, private); | |||||
buf = NULL; | |||||
} | |||||
ASSERT((zio_flags & ZIO_FLAG_SPECULATIVE) || | |||||
rc == 0 || rc != ENOENT); | |||||
} else if (*arc_flags & ARC_FLAG_PREFETCH && | } else if (*arc_flags & ARC_FLAG_PREFETCH && | ||||
refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) { | refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) { | ||||
arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH); | arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH); | ||||
} | } | ||||
DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); | DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); | ||||
arc_access(hdr, hash_lock); | arc_access(hdr, hash_lock); | ||||
if (*arc_flags & ARC_FLAG_PRESCIENT_PREFETCH) | |||||
arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH); | |||||
if (*arc_flags & ARC_FLAG_L2CACHE) | if (*arc_flags & ARC_FLAG_L2CACHE) | ||||
arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); | arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); | ||||
mutex_exit(hash_lock); | mutex_exit(hash_lock); | ||||
ARCSTAT_BUMP(arcstat_hits); | ARCSTAT_BUMP(arcstat_hits); | ||||
ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr), | ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr), | ||||
demand, prefetch, !HDR_ISTYPE_METADATA(hdr), | demand, prefetch, !HDR_ISTYPE_METADATA(hdr), | ||||
data, metadata, hits); | data, metadata, hits); | ||||
if (done) | if (done) | ||||
done(NULL, buf, private); | done(NULL, zb, bp, buf, private); | ||||
} else { | } else { | ||||
uint64_t lsize = BP_GET_LSIZE(bp); | uint64_t lsize = BP_GET_LSIZE(bp); | ||||
uint64_t psize = BP_GET_PSIZE(bp); | uint64_t psize = BP_GET_PSIZE(bp); | ||||
arc_callback_t *acb; | arc_callback_t *acb; | ||||
vdev_t *vd = NULL; | vdev_t *vd = NULL; | ||||
uint64_t addr = 0; | uint64_t addr = 0; | ||||
boolean_t devw = B_FALSE; | boolean_t devw = B_FALSE; | ||||
uint64_t size; | uint64_t size; | ||||
▲ Show 20 Lines • Show All 57 Lines • ▼ Show 20 Lines | if (hdr != NULL && HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_pabd != NULL) { | ||||
* the uncompressed data. | * the uncompressed data. | ||||
*/ | */ | ||||
if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF) { | if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF) { | ||||
zio_flags |= ZIO_FLAG_RAW; | zio_flags |= ZIO_FLAG_RAW; | ||||
} | } | ||||
if (*arc_flags & ARC_FLAG_PREFETCH) | if (*arc_flags & ARC_FLAG_PREFETCH) | ||||
arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH); | arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH); | ||||
if (*arc_flags & ARC_FLAG_PRESCIENT_PREFETCH) | |||||
arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH); | |||||
if (*arc_flags & ARC_FLAG_L2CACHE) | if (*arc_flags & ARC_FLAG_L2CACHE) | ||||
arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); | arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); | ||||
if (BP_GET_LEVEL(bp) > 0) | if (BP_GET_LEVEL(bp) > 0) | ||||
arc_hdr_set_flags(hdr, ARC_FLAG_INDIRECT); | arc_hdr_set_flags(hdr, ARC_FLAG_INDIRECT); | ||||
if (*arc_flags & ARC_FLAG_PREDICTIVE_PREFETCH) | if (*arc_flags & ARC_FLAG_PREDICTIVE_PREFETCH) | ||||
arc_hdr_set_flags(hdr, ARC_FLAG_PREDICTIVE_PREFETCH); | arc_hdr_set_flags(hdr, ARC_FLAG_PREDICTIVE_PREFETCH); | ||||
ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state)); | ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state)); | ||||
Show All 13 Lines | if (HDR_HAS_L2HDR(hdr) && | ||||
/* | /* | ||||
* Lock out L2ARC device removal. | * Lock out L2ARC device removal. | ||||
*/ | */ | ||||
if (vdev_is_dead(vd) || | if (vdev_is_dead(vd) || | ||||
!spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER)) | !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER)) | ||||
vd = NULL; | vd = NULL; | ||||
} | } | ||||
if (priority == ZIO_PRIORITY_ASYNC_READ) | /* | ||||
* We count both async reads and scrub IOs as asynchronous so | |||||
* that both can be upgraded in the event of a cache hit while | |||||
* the read IO is still in-flight. | |||||
*/ | |||||
if (priority == ZIO_PRIORITY_ASYNC_READ || | |||||
priority == ZIO_PRIORITY_SCRUB) | |||||
arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ); | arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ); | ||||
else | else | ||||
arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ); | arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ); | ||||
if (hash_lock != NULL) | |||||
mutex_exit(hash_lock); | |||||
/* | /* | ||||
* At this point, we have a level 1 cache miss. Try again in | * At this point, we have a level 1 cache miss. Try again in | ||||
* L2ARC if possible. | * L2ARC if possible. | ||||
*/ | */ | ||||
ASSERT3U(HDR_GET_LSIZE(hdr), ==, lsize); | ASSERT3U(HDR_GET_LSIZE(hdr), ==, lsize); | ||||
DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp, | DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp, | ||||
uint64_t, lsize, zbookmark_phys_t *, zb); | uint64_t, lsize, zbookmark_phys_t *, zb); | ||||
▲ Show 20 Lines • Show All 64 Lines • ▼ Show 20 Lines | if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) { | ||||
rzio = zio_read_phys(pio, vd, addr, | rzio = zio_read_phys(pio, vd, addr, | ||||
asize, abd, | asize, abd, | ||||
ZIO_CHECKSUM_OFF, | ZIO_CHECKSUM_OFF, | ||||
l2arc_read_done, cb, priority, | l2arc_read_done, cb, priority, | ||||
zio_flags | ZIO_FLAG_DONT_CACHE | | zio_flags | ZIO_FLAG_DONT_CACHE | | ||||
ZIO_FLAG_CANFAIL | | ZIO_FLAG_CANFAIL | | ||||
ZIO_FLAG_DONT_PROPAGATE | | ZIO_FLAG_DONT_PROPAGATE | | ||||
ZIO_FLAG_DONT_RETRY, B_FALSE); | ZIO_FLAG_DONT_RETRY, B_FALSE); | ||||
acb->acb_zio_head = rzio; | |||||
if (hash_lock != NULL) | |||||
mutex_exit(hash_lock); | |||||
DTRACE_PROBE2(l2arc__read, vdev_t *, vd, | DTRACE_PROBE2(l2arc__read, vdev_t *, vd, | ||||
zio_t *, rzio); | zio_t *, rzio); | ||||
ARCSTAT_INCR(arcstat_l2_read_bytes, size); | ARCSTAT_INCR(arcstat_l2_read_bytes, size); | ||||
if (*arc_flags & ARC_FLAG_NOWAIT) { | if (*arc_flags & ARC_FLAG_NOWAIT) { | ||||
zio_nowait(rzio); | zio_nowait(rzio); | ||||
return (0); | return (0); | ||||
} | } | ||||
ASSERT(*arc_flags & ARC_FLAG_WAIT); | ASSERT(*arc_flags & ARC_FLAG_WAIT); | ||||
if (zio_wait(rzio) == 0) | if (zio_wait(rzio) == 0) | ||||
return (0); | return (0); | ||||
/* l2arc read error; goto zio_read() */ | /* l2arc read error; goto zio_read() */ | ||||
if (hash_lock != NULL) | |||||
mutex_enter(hash_lock); | |||||
} else { | } else { | ||||
DTRACE_PROBE1(l2arc__miss, | DTRACE_PROBE1(l2arc__miss, | ||||
arc_buf_hdr_t *, hdr); | arc_buf_hdr_t *, hdr); | ||||
ARCSTAT_BUMP(arcstat_l2_misses); | ARCSTAT_BUMP(arcstat_l2_misses); | ||||
if (HDR_L2_WRITING(hdr)) | if (HDR_L2_WRITING(hdr)) | ||||
ARCSTAT_BUMP(arcstat_l2_rw_clash); | ARCSTAT_BUMP(arcstat_l2_rw_clash); | ||||
spa_config_exit(spa, SCL_L2ARC, vd); | spa_config_exit(spa, SCL_L2ARC, vd); | ||||
} | } | ||||
} else { | } else { | ||||
if (vd != NULL) | if (vd != NULL) | ||||
spa_config_exit(spa, SCL_L2ARC, vd); | spa_config_exit(spa, SCL_L2ARC, vd); | ||||
if (l2arc_ndev != 0) { | if (l2arc_ndev != 0) { | ||||
DTRACE_PROBE1(l2arc__miss, | DTRACE_PROBE1(l2arc__miss, | ||||
arc_buf_hdr_t *, hdr); | arc_buf_hdr_t *, hdr); | ||||
ARCSTAT_BUMP(arcstat_l2_misses); | ARCSTAT_BUMP(arcstat_l2_misses); | ||||
} | } | ||||
} | } | ||||
rzio = zio_read(pio, spa, bp, hdr->b_l1hdr.b_pabd, size, | rzio = zio_read(pio, spa, bp, hdr->b_l1hdr.b_pabd, size, | ||||
arc_read_done, hdr, priority, zio_flags, zb); | arc_read_done, hdr, priority, zio_flags, zb); | ||||
acb->acb_zio_head = rzio; | |||||
if (hash_lock != NULL) | |||||
mutex_exit(hash_lock); | |||||
if (*arc_flags & ARC_FLAG_WAIT) | if (*arc_flags & ARC_FLAG_WAIT) | ||||
return (zio_wait(rzio)); | return (zio_wait(rzio)); | ||||
ASSERT(*arc_flags & ARC_FLAG_NOWAIT); | ASSERT(*arc_flags & ARC_FLAG_NOWAIT); | ||||
zio_nowait(rzio); | zio_nowait(rzio); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 474 Lines • ▼ Show 20 Lines | arc_write_done(zio_t *zio) | ||||
callback->awcb_done(zio, buf, callback->awcb_private); | callback->awcb_done(zio, buf, callback->awcb_private); | ||||
abd_put(zio->io_abd); | abd_put(zio->io_abd); | ||||
kmem_free(callback, sizeof (arc_write_callback_t)); | kmem_free(callback, sizeof (arc_write_callback_t)); | ||||
} | } | ||||
zio_t * | zio_t * | ||||
arc_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, arc_buf_t *buf, | arc_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, arc_buf_t *buf, | ||||
boolean_t l2arc, const zio_prop_t *zp, arc_done_func_t *ready, | boolean_t l2arc, const zio_prop_t *zp, arc_write_done_func_t *ready, | ||||
arc_done_func_t *children_ready, arc_done_func_t *physdone, | arc_write_done_func_t *children_ready, arc_write_done_func_t *physdone, | ||||
arc_done_func_t *done, void *private, zio_priority_t priority, | arc_write_done_func_t *done, void *private, zio_priority_t priority, | ||||
int zio_flags, const zbookmark_phys_t *zb) | int zio_flags, const zbookmark_phys_t *zb) | ||||
{ | { | ||||
arc_buf_hdr_t *hdr = buf->b_hdr; | arc_buf_hdr_t *hdr = buf->b_hdr; | ||||
arc_write_callback_t *callback; | arc_write_callback_t *callback; | ||||
zio_t *zio; | zio_t *zio; | ||||
zio_prop_t localprop = *zp; | zio_prop_t localprop = *zp; | ||||
ASSERT3P(ready, !=, NULL); | ASSERT3P(ready, !=, NULL); | ||||
▲ Show 20 Lines • Show All 409 Lines • ▼ Show 20 Lines | #endif | ||||
mutex_init(&arc_reclaim_lock, NULL, MUTEX_DEFAULT, NULL); | mutex_init(&arc_reclaim_lock, NULL, MUTEX_DEFAULT, NULL); | ||||
cv_init(&arc_reclaim_thread_cv, NULL, CV_DEFAULT, NULL); | cv_init(&arc_reclaim_thread_cv, NULL, CV_DEFAULT, NULL); | ||||
cv_init(&arc_reclaim_waiters_cv, NULL, CV_DEFAULT, NULL); | cv_init(&arc_reclaim_waiters_cv, NULL, CV_DEFAULT, NULL); | ||||
mutex_init(&arc_dnlc_evicts_lock, NULL, MUTEX_DEFAULT, NULL); | mutex_init(&arc_dnlc_evicts_lock, NULL, MUTEX_DEFAULT, NULL); | ||||
cv_init(&arc_dnlc_evicts_cv, NULL, CV_DEFAULT, NULL); | cv_init(&arc_dnlc_evicts_cv, NULL, CV_DEFAULT, NULL); | ||||
/* Convert seconds to clock ticks */ | |||||
arc_min_prefetch_lifespan = 1 * hz; | |||||
/* set min cache to 1/32 of all memory, or arc_abs_min, whichever is more */ | /* set min cache to 1/32 of all memory, or arc_abs_min, whichever is more */ | ||||
arc_c_min = MAX(allmem / 32, arc_abs_min); | arc_c_min = MAX(allmem / 32, arc_abs_min); | ||||
/* set max to 5/8 of all memory, or all but 1GB, whichever is more */ | /* set max to 5/8 of all memory, or all but 1GB, whichever is more */ | ||||
if (allmem >= 1 << 30) | if (allmem >= 1 << 30) | ||||
arc_c_max = allmem - (1 << 30); | arc_c_max = allmem - (1 << 30); | ||||
else | else | ||||
arc_c_max = arc_c_min; | arc_c_max = arc_c_min; | ||||
▲ Show 20 Lines • Show All 1,381 Lines • Show Last 20 Lines |