Changeset View
Changeset View
Standalone View
Standalone View
sys/contrib/openzfs/module/zfs/abd.c
Show First 20 Lines • Show All 99 Lines • ▼ Show 20 Lines | |||||
#include <sys/param.h> | #include <sys/param.h> | ||||
#include <sys/zio.h> | #include <sys/zio.h> | ||||
#include <sys/zfs_context.h> | #include <sys/zfs_context.h> | ||||
#include <sys/zfs_znode.h> | #include <sys/zfs_znode.h> | ||||
/* see block comment above for description */ | /* see block comment above for description */ | ||||
int zfs_abd_scatter_enabled = B_TRUE; | int zfs_abd_scatter_enabled = B_TRUE; | ||||
boolean_t | |||||
abd_is_linear(abd_t *abd) | |||||
{ | |||||
return ((abd->abd_flags & ABD_FLAG_LINEAR) != 0 ? B_TRUE : B_FALSE); | |||||
} | |||||
boolean_t | |||||
abd_is_linear_page(abd_t *abd) | |||||
{ | |||||
return ((abd->abd_flags & ABD_FLAG_LINEAR_PAGE) != 0 ? | |||||
B_TRUE : B_FALSE); | |||||
} | |||||
boolean_t | |||||
abd_is_gang(abd_t *abd) | |||||
{ | |||||
return ((abd->abd_flags & ABD_FLAG_GANG) != 0 ? B_TRUE : | |||||
B_FALSE); | |||||
} | |||||
void | void | ||||
abd_verify(abd_t *abd) | abd_verify(abd_t *abd) | ||||
{ | { | ||||
ASSERT3U(abd->abd_size, >, 0); | ASSERT3U(abd->abd_size, >, 0); | ||||
ASSERT3U(abd->abd_size, <=, SPA_MAXBLOCKSIZE); | ASSERT3U(abd->abd_size, <=, SPA_MAXBLOCKSIZE); | ||||
ASSERT3U(abd->abd_flags, ==, abd->abd_flags & (ABD_FLAG_LINEAR | | ASSERT3U(abd->abd_flags, ==, abd->abd_flags & (ABD_FLAG_LINEAR | | ||||
ABD_FLAG_OWNER | ABD_FLAG_META | ABD_FLAG_MULTI_ZONE | | ABD_FLAG_OWNER | ABD_FLAG_META | ABD_FLAG_MULTI_ZONE | | ||||
ABD_FLAG_MULTI_CHUNK | ABD_FLAG_LINEAR_PAGE | ABD_FLAG_GANG | | ABD_FLAG_MULTI_CHUNK | ABD_FLAG_LINEAR_PAGE | ABD_FLAG_GANG | | ||||
ABD_FLAG_GANG_FREE | ABD_FLAG_ZEROS)); | ABD_FLAG_GANG_FREE | ABD_FLAG_ZEROS | ABD_FLAG_ALLOCD)); | ||||
#ifdef ZFS_DEBUG | |||||
IMPLY(abd->abd_parent != NULL, !(abd->abd_flags & ABD_FLAG_OWNER)); | IMPLY(abd->abd_parent != NULL, !(abd->abd_flags & ABD_FLAG_OWNER)); | ||||
#endif | |||||
IMPLY(abd->abd_flags & ABD_FLAG_META, abd->abd_flags & ABD_FLAG_OWNER); | IMPLY(abd->abd_flags & ABD_FLAG_META, abd->abd_flags & ABD_FLAG_OWNER); | ||||
if (abd_is_linear(abd)) { | if (abd_is_linear(abd)) { | ||||
ASSERT3P(ABD_LINEAR_BUF(abd), !=, NULL); | ASSERT3P(ABD_LINEAR_BUF(abd), !=, NULL); | ||||
} else if (abd_is_gang(abd)) { | } else if (abd_is_gang(abd)) { | ||||
uint_t child_sizes = 0; | uint_t child_sizes = 0; | ||||
for (abd_t *cabd = list_head(&ABD_GANG(abd).abd_gang_chain); | for (abd_t *cabd = list_head(&ABD_GANG(abd).abd_gang_chain); | ||||
cabd != NULL; | cabd != NULL; | ||||
cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd)) { | cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd)) { | ||||
ASSERT(list_link_active(&cabd->abd_gang_link)); | ASSERT(list_link_active(&cabd->abd_gang_link)); | ||||
child_sizes += cabd->abd_size; | child_sizes += cabd->abd_size; | ||||
abd_verify(cabd); | abd_verify(cabd); | ||||
} | } | ||||
ASSERT3U(abd->abd_size, ==, child_sizes); | ASSERT3U(abd->abd_size, ==, child_sizes); | ||||
} else { | } else { | ||||
abd_verify_scatter(abd); | abd_verify_scatter(abd); | ||||
} | } | ||||
} | } | ||||
uint_t | static void | ||||
abd_get_size(abd_t *abd) | abd_init_struct(abd_t *abd) | ||||
{ | { | ||||
abd_verify(abd); | list_link_init(&abd->abd_gang_link); | ||||
return (abd->abd_size); | mutex_init(&abd->abd_mtx, NULL, MUTEX_DEFAULT, NULL); | ||||
abd->abd_flags = 0; | |||||
#ifdef ZFS_DEBUG | |||||
zfs_refcount_create(&abd->abd_children); | |||||
abd->abd_parent = NULL; | |||||
#endif | |||||
abd->abd_size = 0; | |||||
} | } | ||||
static void | |||||
abd_fini_struct(abd_t *abd) | |||||
{ | |||||
mutex_destroy(&abd->abd_mtx); | |||||
ASSERT(!list_link_active(&abd->abd_gang_link)); | |||||
#ifdef ZFS_DEBUG | |||||
zfs_refcount_destroy(&abd->abd_children); | |||||
#endif | |||||
} | |||||
abd_t * | |||||
abd_alloc_struct(size_t size) | |||||
{ | |||||
abd_t *abd = abd_alloc_struct_impl(size); | |||||
abd_init_struct(abd); | |||||
abd->abd_flags |= ABD_FLAG_ALLOCD; | |||||
return (abd); | |||||
} | |||||
void | |||||
abd_free_struct(abd_t *abd) | |||||
{ | |||||
abd_fini_struct(abd); | |||||
abd_free_struct_impl(abd); | |||||
} | |||||
/* | /* | ||||
* Allocate an ABD, along with its own underlying data buffers. Use this if you | * Allocate an ABD, along with its own underlying data buffers. Use this if you | ||||
* don't care whether the ABD is linear or not. | * don't care whether the ABD is linear or not. | ||||
*/ | */ | ||||
abd_t * | abd_t * | ||||
abd_alloc(size_t size, boolean_t is_metadata) | abd_alloc(size_t size, boolean_t is_metadata) | ||||
{ | { | ||||
if (!zfs_abd_scatter_enabled || abd_size_alloc_linear(size)) | if (!zfs_abd_scatter_enabled || abd_size_alloc_linear(size)) | ||||
return (abd_alloc_linear(size, is_metadata)); | return (abd_alloc_linear(size, is_metadata)); | ||||
VERIFY3U(size, <=, SPA_MAXBLOCKSIZE); | VERIFY3U(size, <=, SPA_MAXBLOCKSIZE); | ||||
abd_t *abd = abd_alloc_struct(size); | abd_t *abd = abd_alloc_struct(size); | ||||
abd->abd_flags = ABD_FLAG_OWNER; | abd->abd_flags |= ABD_FLAG_OWNER; | ||||
abd->abd_u.abd_scatter.abd_offset = 0; | abd->abd_u.abd_scatter.abd_offset = 0; | ||||
abd_alloc_chunks(abd, size); | abd_alloc_chunks(abd, size); | ||||
if (is_metadata) { | if (is_metadata) { | ||||
abd->abd_flags |= ABD_FLAG_META; | abd->abd_flags |= ABD_FLAG_META; | ||||
} | } | ||||
abd->abd_size = size; | abd->abd_size = size; | ||||
abd->abd_parent = NULL; | |||||
zfs_refcount_create(&abd->abd_children); | |||||
abd_update_scatter_stats(abd, ABDSTAT_INCR); | abd_update_scatter_stats(abd, ABDSTAT_INCR); | ||||
return (abd); | return (abd); | ||||
} | } | ||||
static void | |||||
abd_free_scatter(abd_t *abd) | |||||
{ | |||||
abd_free_chunks(abd); | |||||
zfs_refcount_destroy(&abd->abd_children); | |||||
abd_update_scatter_stats(abd, ABDSTAT_DECR); | |||||
abd_free_struct(abd); | |||||
} | |||||
static void | |||||
abd_put_gang_abd(abd_t *abd) | |||||
{ | |||||
ASSERT(abd_is_gang(abd)); | |||||
abd_t *cabd; | |||||
while ((cabd = list_remove_head(&ABD_GANG(abd).abd_gang_chain)) | |||||
!= NULL) { | |||||
ASSERT0(cabd->abd_flags & ABD_FLAG_GANG_FREE); | |||||
abd->abd_size -= cabd->abd_size; | |||||
abd_put(cabd); | |||||
} | |||||
ASSERT0(abd->abd_size); | |||||
list_destroy(&ABD_GANG(abd).abd_gang_chain); | |||||
} | |||||
/* | /* | ||||
* Free an ABD allocated from abd_get_offset() or abd_get_from_buf(). Will not | |||||
* free the underlying scatterlist or buffer. | |||||
*/ | |||||
void | |||||
abd_put(abd_t *abd) | |||||
{ | |||||
if (abd == NULL) | |||||
return; | |||||
abd_verify(abd); | |||||
ASSERT(!(abd->abd_flags & ABD_FLAG_OWNER)); | |||||
if (abd->abd_parent != NULL) { | |||||
(void) zfs_refcount_remove_many(&abd->abd_parent->abd_children, | |||||
abd->abd_size, abd); | |||||
} | |||||
if (abd_is_gang(abd)) | |||||
abd_put_gang_abd(abd); | |||||
zfs_refcount_destroy(&abd->abd_children); | |||||
abd_free_struct(abd); | |||||
} | |||||
/* | |||||
* Allocate an ABD that must be linear, along with its own underlying data | * Allocate an ABD that must be linear, along with its own underlying data | ||||
* buffer. Only use this when it would be very annoying to write your ABD | * buffer. Only use this when it would be very annoying to write your ABD | ||||
* consumer with a scattered ABD. | * consumer with a scattered ABD. | ||||
*/ | */ | ||||
abd_t * | abd_t * | ||||
abd_alloc_linear(size_t size, boolean_t is_metadata) | abd_alloc_linear(size_t size, boolean_t is_metadata) | ||||
{ | { | ||||
abd_t *abd = abd_alloc_struct(0); | abd_t *abd = abd_alloc_struct(0); | ||||
VERIFY3U(size, <=, SPA_MAXBLOCKSIZE); | VERIFY3U(size, <=, SPA_MAXBLOCKSIZE); | ||||
abd->abd_flags = ABD_FLAG_LINEAR | ABD_FLAG_OWNER; | abd->abd_flags |= ABD_FLAG_LINEAR | ABD_FLAG_OWNER; | ||||
if (is_metadata) { | if (is_metadata) { | ||||
abd->abd_flags |= ABD_FLAG_META; | abd->abd_flags |= ABD_FLAG_META; | ||||
} | } | ||||
abd->abd_size = size; | abd->abd_size = size; | ||||
abd->abd_parent = NULL; | |||||
zfs_refcount_create(&abd->abd_children); | |||||
if (is_metadata) { | if (is_metadata) { | ||||
ABD_LINEAR_BUF(abd) = zio_buf_alloc(size); | ABD_LINEAR_BUF(abd) = zio_buf_alloc(size); | ||||
} else { | } else { | ||||
ABD_LINEAR_BUF(abd) = zio_data_buf_alloc(size); | ABD_LINEAR_BUF(abd) = zio_data_buf_alloc(size); | ||||
} | } | ||||
abd_update_linear_stats(abd, ABDSTAT_INCR); | abd_update_linear_stats(abd, ABDSTAT_INCR); | ||||
Show All 9 Lines | if (abd_is_linear_page(abd)) { | ||||
return; | return; | ||||
} | } | ||||
if (abd->abd_flags & ABD_FLAG_META) { | if (abd->abd_flags & ABD_FLAG_META) { | ||||
zio_buf_free(ABD_LINEAR_BUF(abd), abd->abd_size); | zio_buf_free(ABD_LINEAR_BUF(abd), abd->abd_size); | ||||
} else { | } else { | ||||
zio_data_buf_free(ABD_LINEAR_BUF(abd), abd->abd_size); | zio_data_buf_free(ABD_LINEAR_BUF(abd), abd->abd_size); | ||||
} | } | ||||
zfs_refcount_destroy(&abd->abd_children); | |||||
abd_update_linear_stats(abd, ABDSTAT_DECR); | abd_update_linear_stats(abd, ABDSTAT_DECR); | ||||
abd_free_struct(abd); | |||||
} | } | ||||
static void | static void | ||||
abd_free_gang_abd(abd_t *abd) | abd_free_gang(abd_t *abd) | ||||
{ | { | ||||
ASSERT(abd_is_gang(abd)); | ASSERT(abd_is_gang(abd)); | ||||
abd_t *cabd = list_head(&ABD_GANG(abd).abd_gang_chain); | abd_t *cabd; | ||||
while (cabd != NULL) { | while ((cabd = list_head(&ABD_GANG(abd).abd_gang_chain)) != NULL) { | ||||
/* | /* | ||||
* We must acquire the child ABDs mutex to ensure that if it | * We must acquire the child ABDs mutex to ensure that if it | ||||
* is being added to another gang ABD we will set the link | * is being added to another gang ABD we will set the link | ||||
* as inactive when removing it from this gang ABD and before | * as inactive when removing it from this gang ABD and before | ||||
* adding it to the other gang ABD. | * adding it to the other gang ABD. | ||||
*/ | */ | ||||
mutex_enter(&cabd->abd_mtx); | mutex_enter(&cabd->abd_mtx); | ||||
ASSERT(list_link_active(&cabd->abd_gang_link)); | ASSERT(list_link_active(&cabd->abd_gang_link)); | ||||
list_remove(&ABD_GANG(abd).abd_gang_chain, cabd); | list_remove(&ABD_GANG(abd).abd_gang_chain, cabd); | ||||
mutex_exit(&cabd->abd_mtx); | mutex_exit(&cabd->abd_mtx); | ||||
abd->abd_size -= cabd->abd_size; | if (cabd->abd_flags & ABD_FLAG_GANG_FREE) | ||||
if (cabd->abd_flags & ABD_FLAG_GANG_FREE) { | |||||
if (cabd->abd_flags & ABD_FLAG_OWNER) | |||||
abd_free(cabd); | abd_free(cabd); | ||||
else | |||||
abd_put(cabd); | |||||
} | } | ||||
cabd = list_head(&ABD_GANG(abd).abd_gang_chain); | |||||
} | |||||
ASSERT0(abd->abd_size); | |||||
list_destroy(&ABD_GANG(abd).abd_gang_chain); | list_destroy(&ABD_GANG(abd).abd_gang_chain); | ||||
zfs_refcount_destroy(&abd->abd_children); | |||||
abd_free_struct(abd); | |||||
} | } | ||||
static void | |||||
abd_free_scatter(abd_t *abd) | |||||
{ | |||||
abd_free_chunks(abd); | |||||
abd_update_scatter_stats(abd, ABDSTAT_DECR); | |||||
} | |||||
/* | /* | ||||
* Free an ABD. Only use this on ABDs allocated with abd_alloc(), | * Free an ABD. Use with any kind of abd: those created with abd_alloc_*() | ||||
* abd_alloc_linear(), or abd_alloc_gang_abd(). | * and abd_get_*(), including abd_get_offset_struct(). | ||||
* | |||||
* If the ABD was created with abd_alloc_*(), the underlying data | |||||
* (scatterlist or linear buffer) will also be freed. (Subject to ownership | |||||
* changes via abd_*_ownership_of_buf().) | |||||
* | |||||
* Unless the ABD was created with abd_get_offset_struct(), the abd_t will | |||||
* also be freed. | |||||
*/ | */ | ||||
void | void | ||||
abd_free(abd_t *abd) | abd_free(abd_t *abd) | ||||
{ | { | ||||
if (abd == NULL) | if (abd == NULL) | ||||
return; | return; | ||||
abd_verify(abd); | abd_verify(abd); | ||||
ASSERT3P(abd->abd_parent, ==, NULL); | #ifdef ZFS_DEBUG | ||||
ASSERT(abd->abd_flags & ABD_FLAG_OWNER); | IMPLY(abd->abd_flags & ABD_FLAG_OWNER, abd->abd_parent == NULL); | ||||
if (abd_is_linear(abd)) | #endif | ||||
if (abd_is_gang(abd)) { | |||||
abd_free_gang(abd); | |||||
} else if (abd_is_linear(abd)) { | |||||
if (abd->abd_flags & ABD_FLAG_OWNER) | |||||
abd_free_linear(abd); | abd_free_linear(abd); | ||||
else if (abd_is_gang(abd)) | } else { | ||||
abd_free_gang_abd(abd); | if (abd->abd_flags & ABD_FLAG_OWNER) | ||||
else | |||||
abd_free_scatter(abd); | abd_free_scatter(abd); | ||||
} | } | ||||
#ifdef ZFS_DEBUG | |||||
if (abd->abd_parent != NULL) { | |||||
(void) zfs_refcount_remove_many(&abd->abd_parent->abd_children, | |||||
abd->abd_size, abd); | |||||
} | |||||
#endif | |||||
abd_fini_struct(abd); | |||||
if (abd->abd_flags & ABD_FLAG_ALLOCD) | |||||
abd_free_struct_impl(abd); | |||||
} | |||||
/* | /* | ||||
* Allocate an ABD of the same format (same metadata flag, same scatterize | * Allocate an ABD of the same format (same metadata flag, same scatterize | ||||
* setting) as another ABD. | * setting) as another ABD. | ||||
*/ | */ | ||||
abd_t * | abd_t * | ||||
abd_alloc_sametype(abd_t *sabd, size_t size) | abd_alloc_sametype(abd_t *sabd, size_t size) | ||||
{ | { | ||||
boolean_t is_metadata = (sabd->abd_flags & ABD_FLAG_META) != 0; | boolean_t is_metadata = (sabd->abd_flags & ABD_FLAG_META) != 0; | ||||
if (abd_is_linear(sabd) && | if (abd_is_linear(sabd) && | ||||
!abd_is_linear_page(sabd)) { | !abd_is_linear_page(sabd)) { | ||||
return (abd_alloc_linear(size, is_metadata)); | return (abd_alloc_linear(size, is_metadata)); | ||||
} else { | } else { | ||||
return (abd_alloc(size, is_metadata)); | return (abd_alloc(size, is_metadata)); | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Create gang ABD that will be the head of a list of ABD's. This is used | * Create gang ABD that will be the head of a list of ABD's. This is used | ||||
* to "chain" scatter/gather lists together when constructing aggregated | * to "chain" scatter/gather lists together when constructing aggregated | ||||
* IO's. To free this abd, abd_free() must be called. | * IO's. To free this abd, abd_free() must be called. | ||||
*/ | */ | ||||
abd_t * | abd_t * | ||||
abd_alloc_gang_abd(void) | abd_alloc_gang(void) | ||||
{ | { | ||||
abd_t *abd; | abd_t *abd = abd_alloc_struct(0); | ||||
abd->abd_flags |= ABD_FLAG_GANG | ABD_FLAG_OWNER; | |||||
abd = abd_alloc_struct(0); | |||||
abd->abd_flags = ABD_FLAG_GANG | ABD_FLAG_OWNER; | |||||
abd->abd_size = 0; | |||||
abd->abd_parent = NULL; | |||||
list_create(&ABD_GANG(abd).abd_gang_chain, | list_create(&ABD_GANG(abd).abd_gang_chain, | ||||
sizeof (abd_t), offsetof(abd_t, abd_gang_link)); | sizeof (abd_t), offsetof(abd_t, abd_gang_link)); | ||||
zfs_refcount_create(&abd->abd_children); | |||||
return (abd); | return (abd); | ||||
} | } | ||||
/* | /* | ||||
* Add a child gang ABD to a parent gang ABDs chained list. | * Add a child gang ABD to a parent gang ABDs chained list. | ||||
*/ | */ | ||||
static void | static void | ||||
abd_gang_add_gang(abd_t *pabd, abd_t *cabd, boolean_t free_on_free) | abd_gang_add_gang(abd_t *pabd, abd_t *cabd, boolean_t free_on_free) | ||||
{ | { | ||||
ASSERT(abd_is_gang(pabd)); | ASSERT(abd_is_gang(pabd)); | ||||
ASSERT(abd_is_gang(cabd)); | ASSERT(abd_is_gang(cabd)); | ||||
if (free_on_free) { | if (free_on_free) { | ||||
/* | /* | ||||
* If the parent is responsible for freeing the child gang | * If the parent is responsible for freeing the child gang | ||||
* ABD we will just splice the childs children ABD list to | * ABD we will just splice the child's children ABD list to | ||||
* the parents list and immediately free the child gang ABD | * the parent's list and immediately free the child gang ABD | ||||
* struct. The parent gang ABDs children from the child gang | * struct. The parent gang ABDs children from the child gang | ||||
* will retain all the free_on_free settings after being | * will retain all the free_on_free settings after being | ||||
* added to the parents list. | * added to the parents list. | ||||
*/ | */ | ||||
pabd->abd_size += cabd->abd_size; | pabd->abd_size += cabd->abd_size; | ||||
list_move_tail(&ABD_GANG(pabd).abd_gang_chain, | list_move_tail(&ABD_GANG(pabd).abd_gang_chain, | ||||
&ABD_GANG(cabd).abd_gang_chain); | &ABD_GANG(cabd).abd_gang_chain); | ||||
ASSERT(list_is_empty(&ABD_GANG(cabd).abd_gang_chain)); | ASSERT(list_is_empty(&ABD_GANG(cabd).abd_gang_chain)); | ||||
abd_verify(pabd); | abd_verify(pabd); | ||||
abd_free_struct(cabd); | abd_free(cabd); | ||||
} else { | } else { | ||||
for (abd_t *child = list_head(&ABD_GANG(cabd).abd_gang_chain); | for (abd_t *child = list_head(&ABD_GANG(cabd).abd_gang_chain); | ||||
child != NULL; | child != NULL; | ||||
child = list_next(&ABD_GANG(cabd).abd_gang_chain, child)) { | child = list_next(&ABD_GANG(cabd).abd_gang_chain, child)) { | ||||
/* | /* | ||||
* We always pass B_FALSE for free_on_free as it is the | * We always pass B_FALSE for free_on_free as it is the | ||||
* original child gang ABDs responsibilty to determine | * original child gang ABDs responsibilty to determine | ||||
* if any of its child ABDs should be free'd on the call | * if any of its child ABDs should be free'd on the call | ||||
Show All 11 Lines | |||||
void | void | ||||
abd_gang_add(abd_t *pabd, abd_t *cabd, boolean_t free_on_free) | abd_gang_add(abd_t *pabd, abd_t *cabd, boolean_t free_on_free) | ||||
{ | { | ||||
ASSERT(abd_is_gang(pabd)); | ASSERT(abd_is_gang(pabd)); | ||||
abd_t *child_abd = NULL; | abd_t *child_abd = NULL; | ||||
/* | /* | ||||
* If the child being added is a gang ABD, we will add the | * If the child being added is a gang ABD, we will add the | ||||
* childs ABDs to the parent gang ABD. This alllows us to account | * child's ABDs to the parent gang ABD. This allows us to account | ||||
* for the offset correctly in the parent gang ABD. | * for the offset correctly in the parent gang ABD. | ||||
*/ | */ | ||||
if (abd_is_gang(cabd)) { | if (abd_is_gang(cabd)) { | ||||
ASSERT(!list_link_active(&cabd->abd_gang_link)); | ASSERT(!list_link_active(&cabd->abd_gang_link)); | ||||
ASSERT(!list_is_empty(&ABD_GANG(cabd).abd_gang_chain)); | ASSERT(!list_is_empty(&ABD_GANG(cabd).abd_gang_chain)); | ||||
return (abd_gang_add_gang(pabd, cabd, free_on_free)); | return (abd_gang_add_gang(pabd, cabd, free_on_free)); | ||||
} | } | ||||
ASSERT(!abd_is_gang(cabd)); | ASSERT(!abd_is_gang(cabd)); | ||||
Show All 10 Lines | abd_gang_add(abd_t *pabd, abd_t *cabd, boolean_t free_on_free) | ||||
if (list_link_active(&cabd->abd_gang_link)) { | if (list_link_active(&cabd->abd_gang_link)) { | ||||
/* | /* | ||||
* If the child ABD is already part of another | * If the child ABD is already part of another | ||||
* gang ABD then we must allocate a new | * gang ABD then we must allocate a new | ||||
* ABD to use a separate link. We mark the newly | * ABD to use a separate link. We mark the newly | ||||
* allocated ABD with ABD_FLAG_GANG_FREE, before | * allocated ABD with ABD_FLAG_GANG_FREE, before | ||||
* adding it to the gang ABD's list, to make the | * adding it to the gang ABD's list, to make the | ||||
* gang ABD aware that it is responsible to call | * gang ABD aware that it is responsible to call | ||||
* abd_put(). We use abd_get_offset() in order | * abd_free(). We use abd_get_offset() in order | ||||
* to just allocate a new ABD but avoid copying the | * to just allocate a new ABD but avoid copying the | ||||
* data over into the newly allocated ABD. | * data over into the newly allocated ABD. | ||||
* | * | ||||
* An ABD may become part of multiple gang ABD's. For | * An ABD may become part of multiple gang ABD's. For | ||||
* example, when writing ditto bocks, the same ABD | * example, when writing ditto bocks, the same ABD | ||||
* is used to write 2 or 3 locations with 2 or 3 | * is used to write 2 or 3 locations with 2 or 3 | ||||
* zio_t's. Each of the zio's may be aggregated with | * zio_t's. Each of the zio's may be aggregated with | ||||
* different adjacent zio's. zio aggregation uses gang | * different adjacent zio's. zio aggregation uses gang | ||||
Show All 40 Lines | for (cabd = list_head(&ABD_GANG(abd).abd_gang_chain); cabd != NULL; | ||||
else | else | ||||
return (cabd); | return (cabd); | ||||
} | } | ||||
VERIFY3P(cabd, !=, NULL); | VERIFY3P(cabd, !=, NULL); | ||||
return (cabd); | return (cabd); | ||||
} | } | ||||
/* | /* | ||||
* Allocate a new ABD to point to offset off of sabd. It shares the underlying | * Allocate a new ABD, using the provided struct (if non-NULL, and if | ||||
* buffer data with sabd. Use abd_put() to free. sabd must not be freed while | * circumstances allow - otherwise allocate the struct). The returned ABD will | ||||
* any derived ABDs exist. | * point to offset off of sabd. It shares the underlying buffer data with sabd. | ||||
* Use abd_free() to free. sabd must not be freed while any derived ABDs exist. | |||||
*/ | */ | ||||
static abd_t * | static abd_t * | ||||
abd_get_offset_impl(abd_t *sabd, size_t off, size_t size) | abd_get_offset_impl(abd_t *abd, abd_t *sabd, size_t off, size_t size) | ||||
{ | { | ||||
abd_t *abd = NULL; | |||||
abd_verify(sabd); | abd_verify(sabd); | ||||
ASSERT3U(off, <=, sabd->abd_size); | ASSERT3U(off + size, <=, sabd->abd_size); | ||||
if (abd_is_linear(sabd)) { | if (abd_is_linear(sabd)) { | ||||
if (abd == NULL) | |||||
abd = abd_alloc_struct(0); | abd = abd_alloc_struct(0); | ||||
/* | /* | ||||
* Even if this buf is filesystem metadata, we only track that | * Even if this buf is filesystem metadata, we only track that | ||||
* if we own the underlying data buffer, which is not true in | * if we own the underlying data buffer, which is not true in | ||||
* this case. Therefore, we don't ever use ABD_FLAG_META here. | * this case. Therefore, we don't ever use ABD_FLAG_META here. | ||||
*/ | */ | ||||
abd->abd_flags = ABD_FLAG_LINEAR; | abd->abd_flags |= ABD_FLAG_LINEAR; | ||||
ABD_LINEAR_BUF(abd) = (char *)ABD_LINEAR_BUF(sabd) + off; | ABD_LINEAR_BUF(abd) = (char *)ABD_LINEAR_BUF(sabd) + off; | ||||
} else if (abd_is_gang(sabd)) { | } else if (abd_is_gang(sabd)) { | ||||
size_t left = size; | size_t left = size; | ||||
abd = abd_alloc_gang_abd(); | if (abd == NULL) { | ||||
abd = abd_alloc_gang(); | |||||
} else { | |||||
abd->abd_flags |= ABD_FLAG_GANG; | |||||
list_create(&ABD_GANG(abd).abd_gang_chain, | |||||
sizeof (abd_t), offsetof(abd_t, abd_gang_link)); | |||||
} | |||||
abd->abd_flags &= ~ABD_FLAG_OWNER; | abd->abd_flags &= ~ABD_FLAG_OWNER; | ||||
for (abd_t *cabd = abd_gang_get_offset(sabd, &off); | for (abd_t *cabd = abd_gang_get_offset(sabd, &off); | ||||
cabd != NULL && left > 0; | cabd != NULL && left > 0; | ||||
cabd = list_next(&ABD_GANG(sabd).abd_gang_chain, cabd)) { | cabd = list_next(&ABD_GANG(sabd).abd_gang_chain, cabd)) { | ||||
int csize = MIN(left, cabd->abd_size - off); | int csize = MIN(left, cabd->abd_size - off); | ||||
abd_t *nabd = abd_get_offset_impl(cabd, off, csize); | abd_t *nabd = abd_get_offset_size(cabd, off, csize); | ||||
abd_gang_add(abd, nabd, B_FALSE); | abd_gang_add(abd, nabd, B_TRUE); | ||||
left -= csize; | left -= csize; | ||||
off = 0; | off = 0; | ||||
} | } | ||||
ASSERT3U(left, ==, 0); | ASSERT3U(left, ==, 0); | ||||
} else { | } else { | ||||
abd = abd_get_offset_scatter(sabd, off); | abd = abd_get_offset_scatter(abd, sabd, off); | ||||
} | } | ||||
ASSERT3P(abd, !=, NULL); | |||||
abd->abd_size = size; | abd->abd_size = size; | ||||
#ifdef ZFS_DEBUG | |||||
abd->abd_parent = sabd; | abd->abd_parent = sabd; | ||||
zfs_refcount_create(&abd->abd_children); | |||||
(void) zfs_refcount_add_many(&sabd->abd_children, abd->abd_size, abd); | (void) zfs_refcount_add_many(&sabd->abd_children, abd->abd_size, abd); | ||||
#endif | |||||
return (abd); | return (abd); | ||||
} | } | ||||
/* | |||||
* Like abd_get_offset_size(), but memory for the abd_t is provided by the | |||||
* caller. Using this routine can improve performance by avoiding the cost | |||||
* of allocating memory for the abd_t struct, and updating the abd stats. | |||||
* Usually, the provided abd is returned, but in some circumstances (FreeBSD, | |||||
* if sabd is scatter and size is more than 2 pages) a new abd_t may need to | |||||
* be allocated. Therefore callers should be careful to use the returned | |||||
* abd_t*. | |||||
*/ | |||||
abd_t * | abd_t * | ||||
abd_get_offset_struct(abd_t *abd, abd_t *sabd, size_t off, size_t size) | |||||
{ | |||||
abd_init_struct(abd); | |||||
return (abd_get_offset_impl(abd, sabd, off, size)); | |||||
} | |||||
abd_t * | |||||
abd_get_offset(abd_t *sabd, size_t off) | abd_get_offset(abd_t *sabd, size_t off) | ||||
{ | { | ||||
size_t size = sabd->abd_size > off ? sabd->abd_size - off : 0; | size_t size = sabd->abd_size > off ? sabd->abd_size - off : 0; | ||||
VERIFY3U(size, >, 0); | VERIFY3U(size, >, 0); | ||||
return (abd_get_offset_impl(sabd, off, size)); | return (abd_get_offset_impl(NULL, sabd, off, size)); | ||||
} | } | ||||
abd_t * | abd_t * | ||||
abd_get_offset_size(abd_t *sabd, size_t off, size_t size) | abd_get_offset_size(abd_t *sabd, size_t off, size_t size) | ||||
{ | { | ||||
ASSERT3U(off + size, <=, sabd->abd_size); | ASSERT3U(off + size, <=, sabd->abd_size); | ||||
return (abd_get_offset_impl(sabd, off, size)); | return (abd_get_offset_impl(NULL, sabd, off, size)); | ||||
} | } | ||||
/* | /* | ||||
* Return a size scatter ABD. In order to free the returned | * Return a size scatter ABD containing only zeros. | ||||
* ABD abd_put() must be called. | |||||
*/ | */ | ||||
abd_t * | abd_t * | ||||
abd_get_zeros(size_t size) | abd_get_zeros(size_t size) | ||||
{ | { | ||||
ASSERT3P(abd_zero_scatter, !=, NULL); | ASSERT3P(abd_zero_scatter, !=, NULL); | ||||
ASSERT3U(size, <=, SPA_MAXBLOCKSIZE); | ASSERT3U(size, <=, SPA_MAXBLOCKSIZE); | ||||
return (abd_get_offset_size(abd_zero_scatter, 0, size)); | return (abd_get_offset_size(abd_zero_scatter, 0, size)); | ||||
} | } | ||||
/* | /* | ||||
* Allocate a linear ABD structure for buf. You must free this with abd_put() | * Allocate a linear ABD structure for buf. | ||||
* since the resulting ABD doesn't own its own buffer. | |||||
*/ | */ | ||||
abd_t * | abd_t * | ||||
abd_get_from_buf(void *buf, size_t size) | abd_get_from_buf(void *buf, size_t size) | ||||
{ | { | ||||
abd_t *abd = abd_alloc_struct(0); | abd_t *abd = abd_alloc_struct(0); | ||||
VERIFY3U(size, <=, SPA_MAXBLOCKSIZE); | VERIFY3U(size, <=, SPA_MAXBLOCKSIZE); | ||||
/* | /* | ||||
* Even if this buf is filesystem metadata, we only track that if we | * Even if this buf is filesystem metadata, we only track that if we | ||||
* own the underlying data buffer, which is not true in this case. | * own the underlying data buffer, which is not true in this case. | ||||
* Therefore, we don't ever use ABD_FLAG_META here. | * Therefore, we don't ever use ABD_FLAG_META here. | ||||
*/ | */ | ||||
abd->abd_flags = ABD_FLAG_LINEAR; | abd->abd_flags |= ABD_FLAG_LINEAR; | ||||
abd->abd_size = size; | abd->abd_size = size; | ||||
abd->abd_parent = NULL; | |||||
zfs_refcount_create(&abd->abd_children); | |||||
ABD_LINEAR_BUF(abd) = buf; | ABD_LINEAR_BUF(abd) = buf; | ||||
return (abd); | return (abd); | ||||
} | } | ||||
/* | /* | ||||
* Get the raw buffer associated with a linear ABD. | * Get the raw buffer associated with a linear ABD. | ||||
Show All 18 Lines | abd_borrow_buf(abd_t *abd, size_t n) | ||||
void *buf; | void *buf; | ||||
abd_verify(abd); | abd_verify(abd); | ||||
ASSERT3U(abd->abd_size, >=, n); | ASSERT3U(abd->abd_size, >=, n); | ||||
if (abd_is_linear(abd)) { | if (abd_is_linear(abd)) { | ||||
buf = abd_to_buf(abd); | buf = abd_to_buf(abd); | ||||
} else { | } else { | ||||
buf = zio_buf_alloc(n); | buf = zio_buf_alloc(n); | ||||
} | } | ||||
#ifdef ZFS_DEBUG | |||||
(void) zfs_refcount_add_many(&abd->abd_children, n, buf); | (void) zfs_refcount_add_many(&abd->abd_children, n, buf); | ||||
#endif | |||||
return (buf); | return (buf); | ||||
} | } | ||||
void * | void * | ||||
abd_borrow_buf_copy(abd_t *abd, size_t n) | abd_borrow_buf_copy(abd_t *abd, size_t n) | ||||
{ | { | ||||
void *buf = abd_borrow_buf(abd, n); | void *buf = abd_borrow_buf(abd, n); | ||||
if (!abd_is_linear(abd)) { | if (!abd_is_linear(abd)) { | ||||
Show All 14 Lines | abd_return_buf(abd_t *abd, void *buf, size_t n) | ||||
abd_verify(abd); | abd_verify(abd); | ||||
ASSERT3U(abd->abd_size, >=, n); | ASSERT3U(abd->abd_size, >=, n); | ||||
if (abd_is_linear(abd)) { | if (abd_is_linear(abd)) { | ||||
ASSERT3P(buf, ==, abd_to_buf(abd)); | ASSERT3P(buf, ==, abd_to_buf(abd)); | ||||
} else { | } else { | ||||
ASSERT0(abd_cmp_buf(abd, buf, n)); | ASSERT0(abd_cmp_buf(abd, buf, n)); | ||||
zio_buf_free(buf, n); | zio_buf_free(buf, n); | ||||
} | } | ||||
#ifdef ZFS_DEBUG | |||||
(void) zfs_refcount_remove_many(&abd->abd_children, n, buf); | (void) zfs_refcount_remove_many(&abd->abd_children, n, buf); | ||||
#endif | |||||
} | } | ||||
void | void | ||||
abd_return_buf_copy(abd_t *abd, void *buf, size_t n) | abd_return_buf_copy(abd_t *abd, void *buf, size_t n) | ||||
{ | { | ||||
if (!abd_is_linear(abd)) { | if (!abd_is_linear(abd)) { | ||||
abd_copy_from_buf(abd, buf, n); | abd_copy_from_buf(abd, buf, n); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 97 Lines • ▼ Show 20 Lines | abd_iterate_func(abd_t *abd, size_t off, size_t size, | ||||
int ret = 0; | int ret = 0; | ||||
if (size == 0) | if (size == 0) | ||||
return (0); | return (0); | ||||
abd_verify(abd); | abd_verify(abd); | ||||
ASSERT3U(off + size, <=, abd->abd_size); | ASSERT3U(off + size, <=, abd->abd_size); | ||||
boolean_t abd_multi = abd_is_gang(abd); | boolean_t gang = abd_is_gang(abd); | ||||
abd_t *c_abd = abd_init_abd_iter(abd, &aiter, off); | abd_t *c_abd = abd_init_abd_iter(abd, &aiter, off); | ||||
while (size > 0) { | while (size > 0) { | ||||
/* If we are at the end of the gang ABD we are done */ | /* If we are at the end of the gang ABD we are done */ | ||||
if (abd_multi && !c_abd) | if (gang && !c_abd) | ||||
break; | break; | ||||
abd_iter_map(&aiter); | abd_iter_map(&aiter); | ||||
size_t len = MIN(aiter.iter_mapsize, size); | size_t len = MIN(aiter.iter_mapsize, size); | ||||
ASSERT3U(len, >, 0); | ASSERT3U(len, >, 0); | ||||
ret = func(aiter.iter_mapaddr, len, private); | ret = func(aiter.iter_mapaddr, len, private); | ||||
▲ Show 20 Lines • Show All 411 Lines • Show Last 20 Lines |