diff --git a/share/man/man4/nda.4 b/share/man/man4/nda.4 --- a/share/man/man4/nda.4 +++ b/share/man/man4/nda.4 @@ -58,6 +58,15 @@ See .Xr nvd 4 when set to 1. +.It Va hw.nvme.nvme_reserved_new_bios +The number of the +.Vt bio +items preallocated and reserved on each device. +These +.Vt bio +items are used to split a single +.Vt bio +into multiple pieces aligned to the optimum block boundaries. .It Va kern.cam.nda.nvd_compat When set to 1, .Xr nvd 4 diff --git a/sys/dev/nvme/nvme_ns.c b/sys/dev/nvme/nvme_ns.c --- a/sys/dev/nvme/nvme_ns.c +++ b/sys/dev/nvme/nvme_ns.c @@ -51,8 +51,9 @@ uint32_t alignment); static void nvme_free_child_bios(int num_bios, struct bio **child_bios); -static struct bio ** nvme_allocate_child_bios(int num_bios); -static struct bio ** nvme_construct_child_bios(struct bio *bp, +static struct bio ** nvme_allocate_child_bios(uma_zone_t uz, int num_bios); +static struct bio ** nvme_construct_child_bios(struct nvme_namespace *ns, + struct bio *bp, uint32_t alignment, int *num_bios); static int nvme_ns_split_bio(struct nvme_namespace *ns, @@ -337,7 +338,7 @@ } static struct bio ** -nvme_allocate_child_bios(int num_bios) +nvme_allocate_child_bios(uma_zone_t uz, int num_bios) { struct bio **child_bios; int err = 0, i; @@ -347,7 +348,7 @@ return (NULL); for (i = 0; i < num_bios; i++) { - child_bios[i] = g_new_bio(); + child_bios[i] = g_new_bio_uz(uz); if (child_bios[i] == NULL) err = ENOMEM; } @@ -361,7 +362,11 @@ } static struct bio ** -nvme_construct_child_bios(struct bio *bp, uint32_t alignment, int *num_bios) +nvme_construct_child_bios( + struct nvme_namespace *ns, + struct bio *bp, + uint32_t alignment, + int *num_bios) { struct bio **child_bios; struct bio *child; @@ -374,7 +379,13 @@ *num_bios = nvme_get_num_segments(bp->bio_offset, bp->bio_bcount, alignment); - child_bios = nvme_allocate_child_bios(*num_bios); + /* + * Inherit the uma(9) zone of the parent bio if this namespace does not + * have one, as splitting a bio is a variation of cloning. + */ + child_bios = nvme_allocate_child_bios( + __predict_true(NULL != ns->zone) ? ns->zone : bp->bio_uz, + *num_bios); if (child_bios == NULL) return (NULL); @@ -424,7 +435,7 @@ struct bio **child_bios; int err, i, num_bios; - child_bios = nvme_construct_child_bios(bp, alignment, &num_bios); + child_bios = nvme_construct_child_bios(ns, bp, alignment, &num_bios); if (child_bios == NULL) return (ENOMEM); @@ -585,6 +596,13 @@ if (vwc_present) ns->flags |= NVME_NS_FLUSH_SUPPORTED; + if (0 != ns->boundary && nvme_reserved_new_bios > 0) { + KASSERT(NULL == ns->zone, ("nvme_ns bio zone created already")); + ns->zone = g_io_new_uz("nvme_ns"); + uma_prealloc(ns->zone, nvme_reserved_new_bios); + uma_zone_reserve(ns->zone, nvme_reserved_new_bios); + } + /* * cdev may have already been created, if we are reconstructing the * namespace after a controller-level reset. @@ -619,4 +637,8 @@ if (ns->cdev != NULL) destroy_dev(ns->cdev); + if (NULL != ns->zone) { + uma_zdestroy(ns->zone); + ns->zone = NULL; + } } diff --git a/sys/dev/nvme/nvme_private.h b/sys/dev/nvme/nvme_private.h --- a/sys/dev/nvme/nvme_private.h +++ b/sys/dev/nvme/nvme_private.h @@ -100,6 +100,7 @@ extern int32_t nvme_retry_count; extern bool nvme_verbose_cmd_dump; +extern int nvme_reserved_new_bios; struct nvme_completion_poll_status { struct nvme_completion cpl; @@ -205,6 +206,7 @@ void *cons_cookie[NVME_MAX_CONSUMERS]; uint32_t boundary; struct mtx lock; + uma_zone_t zone; }; /* diff --git a/sys/dev/nvme/nvme_sysctl.c b/sys/dev/nvme/nvme_sysctl.c --- a/sys/dev/nvme/nvme_sysctl.c +++ b/sys/dev/nvme/nvme_sysctl.c @@ -41,6 +41,7 @@ int nvme_use_nvd = NVME_USE_NVD; bool nvme_verbose_cmd_dump = false; +int __read_mostly nvme_reserved_new_bios = 65536; SYSCTL_NODE(_hw, OID_AUTO, nvme, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "NVMe sysctl tunables"); @@ -49,6 +50,9 @@ SYSCTL_BOOL(_hw_nvme, OID_AUTO, verbose_cmd_dump, CTLFLAG_RWTUN, &nvme_verbose_cmd_dump, 0, "enable verbose command printing when a command fails"); +SYSCTL_INT(_hw_nvme, OID_AUTO, nvme_reserved_new_bios, CTLFLAG_RWTUN, + &nvme_reserved_new_bios, 0, + "Number of reserved new nvme namespace bios for non-blocking allocation"); static void nvme_dump_queue(struct nvme_qpair *qpair)