Differential D28677 Diff 83996 sys/contrib/openzfs/module/os/freebsd/zfs/abd_os.c

Changeset View

Standalone View

sys/contrib/openzfs/module/os/freebsd/zfs/abd_os.c

Show First 20 Lines • Show All 196 Lines • ▼ Show 20 Lines	abd_free_chunks(abd_t *abd)

n = abd_scatter_chunkcnt(abd);		n = abd_scatter_chunkcnt(abd);
for (i = 0; i < n; i++) {		for (i = 0; i < n; i++) {
abd_free_chunk(ABD_SCATTER(abd).abd_chunks[i]);		abd_free_chunk(ABD_SCATTER(abd).abd_chunks[i]);
}		}
}		}

abd_t *		abd_t *
abd_alloc_struct(size_t size)		abd_alloc_struct_impl(size_t size)
{		{
uint_t chunkcnt = abd_chunkcnt_for_bytes(size);		uint_t chunkcnt = abd_chunkcnt_for_bytes(size);
/*		/*
* In the event we are allocating a gang ABD, the size passed in		* In the event we are allocating a gang ABD, the size passed in
* will be 0. We must make sure to set abd_size to the size of an		* will be 0. We must make sure to set abd_size to the size of an
* ABD struct as opposed to an ABD scatter with 0 chunks. The gang		* ABD struct as opposed to an ABD scatter with 0 chunks. The gang
* ABD struct allocation accounts for an additional 24 bytes over		* ABD struct allocation accounts for an additional 24 bytes over
* a scatter ABD with 0 chunks.		* a scatter ABD with 0 chunks.
*/		*/
size_t abd_size = MAX(sizeof (abd_t),		size_t abd_size = MAX(sizeof (abd_t),
offsetof(abd_t, abd_u.abd_scatter.abd_chunks[chunkcnt]));		offsetof(abd_t, abd_u.abd_scatter.abd_chunks[chunkcnt]));
abd_t *abd = kmem_alloc(abd_size, KM_PUSHPAGE);		abd_t *abd = kmem_alloc(abd_size, KM_PUSHPAGE);
ASSERT3P(abd, !=, NULL);		ASSERT3P(abd, !=, NULL);
list_link_init(&abd->abd_gang_link);
mutex_init(&abd->abd_mtx, NULL, MUTEX_DEFAULT, NULL);
ABDSTAT_INCR(abdstat_struct_size, abd_size);		ABDSTAT_INCR(abdstat_struct_size, abd_size);

return (abd);		return (abd);
}		}

void		void
abd_free_struct(abd_t *abd)		abd_free_struct_impl(abd_t *abd)
{		{
uint_t chunkcnt = abd_is_linear(abd) \|\| abd_is_gang(abd) ? 0 :		uint_t chunkcnt = abd_is_linear(abd) \|\| abd_is_gang(abd) ? 0 :
abd_scatter_chunkcnt(abd);		abd_scatter_chunkcnt(abd);
ssize_t size = MAX(sizeof (abd_t),		ssize_t size = MAX(sizeof (abd_t),
offsetof(abd_t, abd_u.abd_scatter.abd_chunks[chunkcnt]));		offsetof(abd_t, abd_u.abd_scatter.abd_chunks[chunkcnt]));
mutex_destroy(&abd->abd_mtx);
ASSERT(!list_link_active(&abd->abd_gang_link));
kmem_free(abd, size);		kmem_free(abd, size);
ABDSTAT_INCR(abdstat_struct_size, -size);		ABDSTAT_INCR(abdstat_struct_size, -size);
}		}

/*		/*
* Allocate scatter ABD of size SPA_MAXBLOCKSIZE, where		* Allocate scatter ABD of size SPA_MAXBLOCKSIZE, where
* each chunk in the scatterlist will be set to abd_zero_buf.		* each chunk in the scatterlist will be set to abd_zero_buf.
*/		*/
static void		static void
abd_alloc_zero_scatter(void)		abd_alloc_zero_scatter(void)
{		{
uint_t i, n;		uint_t i, n;

n = abd_chunkcnt_for_bytes(SPA_MAXBLOCKSIZE);		n = abd_chunkcnt_for_bytes(SPA_MAXBLOCKSIZE);
abd_zero_buf = kmem_zalloc(zfs_abd_chunk_size, KM_SLEEP);		abd_zero_buf = kmem_zalloc(zfs_abd_chunk_size, KM_SLEEP);
abd_zero_scatter = abd_alloc_struct(SPA_MAXBLOCKSIZE);		abd_zero_scatter = abd_alloc_struct(SPA_MAXBLOCKSIZE);

abd_zero_scatter->abd_flags = ABD_FLAG_OWNER \| ABD_FLAG_ZEROS;		abd_zero_scatter->abd_flags \|= ABD_FLAG_OWNER \| ABD_FLAG_ZEROS;
abd_zero_scatter->abd_size = SPA_MAXBLOCKSIZE;		abd_zero_scatter->abd_size = SPA_MAXBLOCKSIZE;
abd_zero_scatter->abd_parent = NULL;
zfs_refcount_create(&abd_zero_scatter->abd_children);

ABD_SCATTER(abd_zero_scatter).abd_offset = 0;		ABD_SCATTER(abd_zero_scatter).abd_offset = 0;
ABD_SCATTER(abd_zero_scatter).abd_chunk_size =		ABD_SCATTER(abd_zero_scatter).abd_chunk_size =
zfs_abd_chunk_size;		zfs_abd_chunk_size;

for (i = 0; i < n; i++) {		for (i = 0; i < n; i++) {
ABD_SCATTER(abd_zero_scatter).abd_chunks[i] =		ABD_SCATTER(abd_zero_scatter).abd_chunks[i] =
abd_zero_buf;		abd_zero_buf;
}		}

ABDSTAT_BUMP(abdstat_scatter_cnt);		ABDSTAT_BUMP(abdstat_scatter_cnt);
ABDSTAT_INCR(abdstat_scatter_data_size, zfs_abd_chunk_size);		ABDSTAT_INCR(abdstat_scatter_data_size, zfs_abd_chunk_size);
}		}

static void		static void
abd_free_zero_scatter(void)		abd_free_zero_scatter(void)
{		{
zfs_refcount_destroy(&abd_zero_scatter->abd_children);
ABDSTAT_BUMPDOWN(abdstat_scatter_cnt);		ABDSTAT_BUMPDOWN(abdstat_scatter_cnt);
ABDSTAT_INCR(abdstat_scatter_data_size, -(int)zfs_abd_chunk_size);		ABDSTAT_INCR(abdstat_scatter_data_size, -(int)zfs_abd_chunk_size);

abd_free_struct(abd_zero_scatter);		abd_free_struct(abd_zero_scatter);
abd_zero_scatter = NULL;		abd_zero_scatter = NULL;
kmem_free(abd_zero_buf, zfs_abd_chunk_size);		kmem_free(abd_zero_buf, zfs_abd_chunk_size);
}		}

▲ Show 20 Lines • Show All 68 Lines • ▼ Show 20 Lines	abd_alloc_scatter_offset_chunkcnt(size_t chunkcnt)
list_link_init(&abd->abd_gang_link);		list_link_init(&abd->abd_gang_link);
mutex_init(&abd->abd_mtx, NULL, MUTEX_DEFAULT, NULL);		mutex_init(&abd->abd_mtx, NULL, MUTEX_DEFAULT, NULL);
ABDSTAT_INCR(abdstat_struct_size, abd_size);		ABDSTAT_INCR(abdstat_struct_size, abd_size);

return (abd);		return (abd);
}		}

abd_t *		abd_t *
abd_get_offset_scatter(abd_t *sabd, size_t off)		abd_get_offset_scatter(abd_t abd, abd_t sabd, size_t off)
{		{
abd_t *abd = NULL;

abd_verify(sabd);		abd_verify(sabd);
ASSERT3U(off, <=, sabd->abd_size);		ASSERT3U(off, <=, sabd->abd_size);

size_t new_offset = ABD_SCATTER(sabd).abd_offset + off;		size_t new_offset = ABD_SCATTER(sabd).abd_offset + off;
uint_t chunkcnt = abd_scatter_chunkcnt(sabd) -		uint_t chunkcnt = abd_scatter_chunkcnt(sabd) -
(new_offset / zfs_abd_chunk_size);		(new_offset / zfs_abd_chunk_size);

abd = abd_alloc_scatter_offset_chunkcnt(chunkcnt);		/*
		* If an abd struct is provided, it is only the minimum size. If we
		* need additional chunks, we need to allocate a new struct.
		*/
		if (abd != NULL &&
		offsetof(abd_t, abd_u.abd_scatter.abd_chunks[chunkcnt]) >
		sizeof (abd_t)) {
		abd = NULL;
		}

		if (abd == NULL)
		abd = abd_alloc_struct(chunkcnt * zfs_abd_chunk_size);

/*		/*
* Even if this buf is filesystem metadata, we only track that		* Even if this buf is filesystem metadata, we only track that
* if we own the underlying data buffer, which is not true in		* if we own the underlying data buffer, which is not true in
* this case. Therefore, we don't ever use ABD_FLAG_META here.		* this case. Therefore, we don't ever use ABD_FLAG_META here.
*/		*/
abd->abd_flags = 0;

ABD_SCATTER(abd).abd_offset = new_offset % zfs_abd_chunk_size;		ABD_SCATTER(abd).abd_offset = new_offset % zfs_abd_chunk_size;
ABD_SCATTER(abd).abd_chunk_size = zfs_abd_chunk_size;		ABD_SCATTER(abd).abd_chunk_size = zfs_abd_chunk_size;

/* Copy the scatterlist starting at the correct offset */		/* Copy the scatterlist starting at the correct offset */
(void) memcpy(&ABD_SCATTER(abd).abd_chunks,		(void) memcpy(&ABD_SCATTER(abd).abd_chunks,
&ABD_SCATTER(sabd).abd_chunks[new_offset /		&ABD_SCATTER(sabd).abd_chunks[new_offset /
zfs_abd_chunk_size],		zfs_abd_chunk_size],
▲ Show 20 Lines • Show All 121 Lines • Show Last 20 Lines