Changeset View
Changeset View
Standalone View
Standalone View
sys/contrib/openzfs/module/zfs/arc.c
- This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 3,059 Lines • ▼ Show 20 Lines | arc_unshare_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf) | ||||
/* | /* | ||||
* We are no longer sharing this buffer so we need | * We are no longer sharing this buffer so we need | ||||
* to transfer its ownership to the rightful owner. | * to transfer its ownership to the rightful owner. | ||||
*/ | */ | ||||
zfs_refcount_transfer_ownership_many(&hdr->b_l1hdr.b_state->arcs_size, | zfs_refcount_transfer_ownership_many(&hdr->b_l1hdr.b_state->arcs_size, | ||||
arc_hdr_size(hdr), hdr, buf); | arc_hdr_size(hdr), hdr, buf); | ||||
arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); | arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); | ||||
abd_release_ownership_of_buf(hdr->b_l1hdr.b_pabd); | abd_release_ownership_of_buf(hdr->b_l1hdr.b_pabd); | ||||
abd_put(hdr->b_l1hdr.b_pabd); | abd_free(hdr->b_l1hdr.b_pabd); | ||||
hdr->b_l1hdr.b_pabd = NULL; | hdr->b_l1hdr.b_pabd = NULL; | ||||
buf->b_flags &= ~ARC_BUF_FLAG_SHARED; | buf->b_flags &= ~ARC_BUF_FLAG_SHARED; | ||||
/* | /* | ||||
* Since the buffer is no longer shared between | * Since the buffer is no longer shared between | ||||
* the arc buf and the hdr, count it as overhead. | * the arc buf and the hdr, count it as overhead. | ||||
*/ | */ | ||||
ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr)); | ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr)); | ||||
▲ Show 20 Lines • Show All 1,081 Lines • ▼ Show 20 Lines | arc_evict_state_impl(multilist_t *ml, int idx, arc_buf_hdr_t *marker, | ||||
* | * | ||||
* Only wake when there's sufficient free memory in the system | * Only wake when there's sufficient free memory in the system | ||||
* (specifically, arc_sys_free/2, which by default is a bit more than | * (specifically, arc_sys_free/2, which by default is a bit more than | ||||
* 1/64th of RAM). See the comments in arc_wait_for_eviction(). | * 1/64th of RAM). See the comments in arc_wait_for_eviction(). | ||||
*/ | */ | ||||
mutex_enter(&arc_evict_lock); | mutex_enter(&arc_evict_lock); | ||||
arc_evict_count += bytes_evicted; | arc_evict_count += bytes_evicted; | ||||
if ((int64_t)(arc_free_memory() - arc_sys_free / 2) > 0) { | if (arc_free_memory() > arc_sys_free / 2) { | ||||
arc_evict_waiter_t *aw; | arc_evict_waiter_t *aw; | ||||
while ((aw = list_head(&arc_evict_waiters)) != NULL && | while ((aw = list_head(&arc_evict_waiters)) != NULL && | ||||
aw->aew_count <= arc_evict_count) { | aw->aew_count <= arc_evict_count) { | ||||
list_remove(&arc_evict_waiters, aw); | list_remove(&arc_evict_waiters, aw); | ||||
cv_broadcast(&aw->aew_cv); | cv_broadcast(&aw->aew_cv); | ||||
} | } | ||||
} | } | ||||
arc_set_need_free(); | arc_set_need_free(); | ||||
▲ Show 20 Lines • Show All 1,062 Lines • ▼ Show 20 Lines | if (arc_is_overflowing()) { | ||||
arc_evict_needed = B_TRUE; | arc_evict_needed = B_TRUE; | ||||
zthr_wakeup(arc_evict_zthr); | zthr_wakeup(arc_evict_zthr); | ||||
if (amount != 0) { | if (amount != 0) { | ||||
arc_evict_waiter_t aw; | arc_evict_waiter_t aw; | ||||
list_link_init(&aw.aew_node); | list_link_init(&aw.aew_node); | ||||
cv_init(&aw.aew_cv, NULL, CV_DEFAULT, NULL); | cv_init(&aw.aew_cv, NULL, CV_DEFAULT, NULL); | ||||
uint64_t last_count = 0; | |||||
if (!list_is_empty(&arc_evict_waiters)) { | |||||
arc_evict_waiter_t *last = | arc_evict_waiter_t *last = | ||||
list_tail(&arc_evict_waiters); | list_tail(&arc_evict_waiters); | ||||
if (last != NULL) { | last_count = last->aew_count; | ||||
ASSERT3U(last->aew_count, >, arc_evict_count); | |||||
aw.aew_count = last->aew_count + amount; | |||||
} else { | |||||
aw.aew_count = arc_evict_count + amount; | |||||
} | } | ||||
/* | |||||
* Note, the last waiter's count may be less than | |||||
* arc_evict_count if we are low on memory in which | |||||
* case arc_evict_state_impl() may have deferred | |||||
* wakeups (but still incremented arc_evict_count). | |||||
*/ | |||||
aw.aew_count = | |||||
MAX(last_count, arc_evict_count) + amount; | |||||
list_insert_tail(&arc_evict_waiters, &aw); | list_insert_tail(&arc_evict_waiters, &aw); | ||||
arc_set_need_free(); | arc_set_need_free(); | ||||
DTRACE_PROBE3(arc__wait__for__eviction, | DTRACE_PROBE3(arc__wait__for__eviction, | ||||
uint64_t, amount, | uint64_t, amount, | ||||
uint64_t, arc_evict_count, | uint64_t, arc_evict_count, | ||||
▲ Show 20 Lines • Show All 1,775 Lines • ▼ Show 20 Lines | if (!HDR_EMPTY(hdr)) { | ||||
mutex_exit(hash_lock); | mutex_exit(hash_lock); | ||||
} else { | } else { | ||||
arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); | arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); | ||||
} | } | ||||
ASSERT(!zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); | ASSERT(!zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); | ||||
callback->awcb_done(zio, buf, callback->awcb_private); | callback->awcb_done(zio, buf, callback->awcb_private); | ||||
abd_put(zio->io_abd); | abd_free(zio->io_abd); | ||||
kmem_free(callback, sizeof (arc_write_callback_t)); | kmem_free(callback, sizeof (arc_write_callback_t)); | ||||
} | } | ||||
zio_t * | zio_t * | ||||
arc_write(zio_t *pio, spa_t *spa, uint64_t txg, | arc_write(zio_t *pio, spa_t *spa, uint64_t txg, | ||||
blkptr_t *bp, arc_buf_t *buf, boolean_t l2arc, | blkptr_t *bp, arc_buf_t *buf, boolean_t l2arc, | ||||
const zio_prop_t *zp, arc_write_done_func_t *ready, | const zio_prop_t *zp, arc_write_done_func_t *ready, | ||||
arc_write_done_func_t *children_ready, arc_write_done_func_t *physdone, | arc_write_done_func_t *children_ready, arc_write_done_func_t *physdone, | ||||
▲ Show 20 Lines • Show All 1,979 Lines • ▼ Show 20 Lines | |||||
static void | static void | ||||
l2arc_blk_fetch_done(zio_t *zio) | l2arc_blk_fetch_done(zio_t *zio) | ||||
{ | { | ||||
l2arc_read_callback_t *cb; | l2arc_read_callback_t *cb; | ||||
cb = zio->io_private; | cb = zio->io_private; | ||||
if (cb->l2rcb_abd != NULL) | if (cb->l2rcb_abd != NULL) | ||||
abd_put(cb->l2rcb_abd); | abd_free(cb->l2rcb_abd); | ||||
kmem_free(cb, sizeof (l2arc_read_callback_t)); | kmem_free(cb, sizeof (l2arc_read_callback_t)); | ||||
} | } | ||||
/* | /* | ||||
* Find and write ARC buffers to the L2ARC device. | * Find and write ARC buffers to the L2ARC device. | ||||
* | * | ||||
* An ARC_FLAG_L2_WRITING flag is set so that the L2ARC buffers are not valid | * An ARC_FLAG_L2_WRITING flag is set so that the L2ARC buffers are not valid | ||||
* for reading until they have completed writing. | * for reading until they have completed writing. | ||||
Show All 21 Lines | l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz) | ||||
write_lsize = write_asize = write_psize = 0; | write_lsize = write_asize = write_psize = 0; | ||||
full = B_FALSE; | full = B_FALSE; | ||||
head = kmem_cache_alloc(hdr_l2only_cache, KM_PUSHPAGE); | head = kmem_cache_alloc(hdr_l2only_cache, KM_PUSHPAGE); | ||||
arc_hdr_set_flags(head, ARC_FLAG_L2_WRITE_HEAD | ARC_FLAG_HAS_L2HDR); | arc_hdr_set_flags(head, ARC_FLAG_L2_WRITE_HEAD | ARC_FLAG_HAS_L2HDR); | ||||
/* | /* | ||||
* Copy buffers for L2ARC writing. | * Copy buffers for L2ARC writing. | ||||
*/ | */ | ||||
for (int try = 0; try < L2ARC_FEED_TYPES; try++) { | for (int pass = 0; pass < L2ARC_FEED_TYPES; pass++) { | ||||
/* | /* | ||||
* If try == 1 or 3, we cache MRU metadata and data | * If pass == 1 or 3, we cache MRU metadata and data | ||||
* respectively. | * respectively. | ||||
*/ | */ | ||||
if (l2arc_mfuonly) { | if (l2arc_mfuonly) { | ||||
if (try == 1 || try == 3) | if (pass == 1 || pass == 3) | ||||
continue; | continue; | ||||
} | } | ||||
multilist_sublist_t *mls = l2arc_sublist_lock(try); | multilist_sublist_t *mls = l2arc_sublist_lock(pass); | ||||
uint64_t passed_sz = 0; | uint64_t passed_sz = 0; | ||||
VERIFY3P(mls, !=, NULL); | VERIFY3P(mls, !=, NULL); | ||||
/* | /* | ||||
* L2ARC fast warmup. | * L2ARC fast warmup. | ||||
* | * | ||||
* Until the ARC is warm and starts to evict, read from the | * Until the ARC is warm and starts to evict, read from the | ||||
▲ Show 20 Lines • Show All 909 Lines • ▼ Show 20 Lines | l2arc_dev_hdr_read(l2arc_dev_t *dev) | ||||
err = zio_wait(zio_read_phys(NULL, dev->l2ad_vdev, | err = zio_wait(zio_read_phys(NULL, dev->l2ad_vdev, | ||||
VDEV_LABEL_START_SIZE, l2dhdr_asize, abd, | VDEV_LABEL_START_SIZE, l2dhdr_asize, abd, | ||||
ZIO_CHECKSUM_LABEL, NULL, NULL, ZIO_PRIORITY_SYNC_READ, | ZIO_CHECKSUM_LABEL, NULL, NULL, ZIO_PRIORITY_SYNC_READ, | ||||
ZIO_FLAG_DONT_CACHE | ZIO_FLAG_CANFAIL | | ZIO_FLAG_DONT_CACHE | ZIO_FLAG_CANFAIL | | ||||
ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY | | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY | | ||||
ZIO_FLAG_SPECULATIVE, B_FALSE)); | ZIO_FLAG_SPECULATIVE, B_FALSE)); | ||||
abd_put(abd); | abd_free(abd); | ||||
if (err != 0) { | if (err != 0) { | ||||
ARCSTAT_BUMP(arcstat_l2_rebuild_abort_dh_errors); | ARCSTAT_BUMP(arcstat_l2_rebuild_abort_dh_errors); | ||||
zfs_dbgmsg("L2ARC IO error (%d) while reading device header, " | zfs_dbgmsg("L2ARC IO error (%d) while reading device header, " | ||||
"vdev guid: %llu", err, dev->l2ad_vdev->vdev_guid); | "vdev guid: %llu", err, dev->l2ad_vdev->vdev_guid); | ||||
return (err); | return (err); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 351 Lines • ▼ Show 20 Lines | if (dev->l2ad_first) | ||||
l2dhdr->dh_flags |= L2ARC_DEV_HDR_EVICT_FIRST; | l2dhdr->dh_flags |= L2ARC_DEV_HDR_EVICT_FIRST; | ||||
abd = abd_get_from_buf(l2dhdr, l2dhdr_asize); | abd = abd_get_from_buf(l2dhdr, l2dhdr_asize); | ||||
err = zio_wait(zio_write_phys(NULL, dev->l2ad_vdev, | err = zio_wait(zio_write_phys(NULL, dev->l2ad_vdev, | ||||
VDEV_LABEL_START_SIZE, l2dhdr_asize, abd, ZIO_CHECKSUM_LABEL, NULL, | VDEV_LABEL_START_SIZE, l2dhdr_asize, abd, ZIO_CHECKSUM_LABEL, NULL, | ||||
NULL, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_CANFAIL, B_FALSE)); | NULL, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_CANFAIL, B_FALSE)); | ||||
abd_put(abd); | abd_free(abd); | ||||
if (err != 0) { | if (err != 0) { | ||||
zfs_dbgmsg("L2ARC IO error (%d) while writing device header, " | zfs_dbgmsg("L2ARC IO error (%d) while writing device header, " | ||||
"vdev guid: %llu", err, dev->l2ad_vdev->vdev_guid); | "vdev guid: %llu", err, dev->l2ad_vdev->vdev_guid); | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 72 Lines • ▼ Show 20 Lines | L2BLK_SET_COMPRESS( | ||||
(&l2dhdr->dh_start_lbps[0])->lbp_prop, | (&l2dhdr->dh_start_lbps[0])->lbp_prop, | ||||
ZIO_COMPRESS_OFF); | ZIO_COMPRESS_OFF); | ||||
} | } | ||||
/* checksum what we're about to write */ | /* checksum what we're about to write */ | ||||
fletcher_4_native(tmpbuf, asize, NULL, | fletcher_4_native(tmpbuf, asize, NULL, | ||||
&l2dhdr->dh_start_lbps[0].lbp_cksum); | &l2dhdr->dh_start_lbps[0].lbp_cksum); | ||||
abd_put(abd_buf->abd); | abd_free(abd_buf->abd); | ||||
/* perform the write itself */ | /* perform the write itself */ | ||||
abd_buf->abd = abd_get_from_buf(tmpbuf, sizeof (*lb)); | abd_buf->abd = abd_get_from_buf(tmpbuf, sizeof (*lb)); | ||||
abd_take_ownership_of_buf(abd_buf->abd, B_TRUE); | abd_take_ownership_of_buf(abd_buf->abd, B_TRUE); | ||||
wzio = zio_write_phys(pio, dev->l2ad_vdev, dev->l2ad_hand, | wzio = zio_write_phys(pio, dev->l2ad_vdev, dev->l2ad_hand, | ||||
asize, abd_buf->abd, ZIO_CHECKSUM_OFF, NULL, NULL, | asize, abd_buf->abd, ZIO_CHECKSUM_OFF, NULL, NULL, | ||||
ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_CANFAIL, B_FALSE); | ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_CANFAIL, B_FALSE); | ||||
DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev, zio_t *, wzio); | DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev, zio_t *, wzio); | ||||
▲ Show 20 Lines • Show All 283 Lines • Show Last 20 Lines |