diff --git a/sys/dev/nvme/nvme_private.h b/sys/dev/nvme/nvme_private.h --- a/sys/dev/nvme/nvme_private.h +++ b/sys/dev/nvme/nvme_private.h @@ -116,10 +116,18 @@ #define NVME_REQUEST_VADDR 1 #define NVME_REQUEST_NULL 2 /* For requests with no payload. */ -#define NVME_REQUEST_UIO 3 #define NVME_REQUEST_BIO 4 #define NVME_REQUEST_CCB 5 +struct nvme_request; +struct nvme_qpair; +struct nvme_tracker; +typedef int (*nvme_request_load_dma_t)(struct nvme_request *, struct nvme_qpair *, struct nvme_tracker *); +int nvme_req_load_dma_vaddr(struct nvme_request *, struct nvme_qpair *, struct nvme_tracker *); +int nvme_req_load_dma_null(struct nvme_request *, struct nvme_qpair *, struct nvme_tracker *); +int nvme_req_load_dma_bio(struct nvme_request *, struct nvme_qpair *, struct nvme_tracker *); +void nvme_payload_map(void *arg, bus_dma_segment_t *seg, int nseg, int error); + struct nvme_request { struct nvme_command cmd; struct nvme_qpair *qpair; @@ -132,6 +140,7 @@ bool timeout; nvme_cb_fn_t cb_fn; void *cb_arg; + nvme_request_load_dma_t payload_dma_map; int32_t retries; STAILQ_ENTRY(nvme_request) stailq; }; @@ -524,6 +533,7 @@ req->type = NVME_REQUEST_VADDR; req->u.payload = payload; req->payload_size = payload_size; + req->payload_dma_map = nvme_req_load_dma_vaddr; } return (req); } @@ -534,8 +544,10 @@ struct nvme_request *req; req = _nvme_allocate_request(cb_fn, cb_arg); - if (req != NULL) + if (req != NULL) { req->type = NVME_REQUEST_NULL; + req->payload_dma_map = nvme_req_load_dma_null; + } return (req); } @@ -548,24 +560,11 @@ if (req != NULL) { req->type = NVME_REQUEST_BIO; req->u.bio = bio; + req->payload_dma_map = nvme_req_load_dma_bio; } return (req); } -static __inline struct nvme_request * -nvme_allocate_request_ccb(union ccb *ccb, nvme_cb_fn_t cb_fn, void *cb_arg) -{ - struct nvme_request *req; - - req = _nvme_allocate_request(cb_fn, cb_arg); - if (req != NULL) { - req->type = NVME_REQUEST_CCB; - req->u.payload = ccb; - } - - return (req); -} - #define nvme_free_request(req) free(req, M_NVME) void nvme_notify_async_consumers(struct nvme_controller *ctrlr, diff --git a/sys/dev/nvme/nvme_qpair.c b/sys/dev/nvme/nvme_qpair.c --- a/sys/dev/nvme/nvme_qpair.c +++ b/sys/dev/nvme/nvme_qpair.c @@ -98,6 +98,49 @@ { 0xFFFF, "IO COMMAND" } }; +int +nvme_req_load_dma_vaddr(struct nvme_request *req, struct nvme_qpair *qpair, struct nvme_tracker *tr) +{ + int err; + + KASSERT(req->payload_size <= qpair->ctrlr->max_xfer_size, + ("payload_size (%d) exceeds max_xfer_size (%d)\n", + req->payload_size, qpair->ctrlr->max_xfer_size)); + err = bus_dmamap_load(tr->qpair->dma_tag_payload, + tr->payload_dma_map, req->u.payload, req->payload_size, + nvme_payload_map, tr, 0); + if (err != 0) + nvme_printf(qpair->ctrlr, + "bus_dmamap_load returned 0x%x!\n", err); + return (err); +} + +int +nvme_req_load_dma_null(struct nvme_request *req, struct nvme_qpair *qpair, struct nvme_tracker *tr) +{ + nvme_qpair_submit_tracker(tr->qpair, tr); + return (0); +} + +int +nvme_req_load_dma_bio(struct nvme_request *req, struct nvme_qpair *qpair, struct nvme_tracker *tr) +{ + int err; + + KASSERT(req->u.bio->bio_bcount <= qpair->ctrlr->max_xfer_size, + ("bio->bio_bcount (%jd) exceeds max_xfer_size (%d)\n", + (intmax_t)req->u.bio->bio_bcount, + qpair->ctrlr->max_xfer_size)); + err = bus_dmamap_load_bio(tr->qpair->dma_tag_payload, + tr->payload_dma_map, req->u.bio, nvme_payload_map, tr, 0); + if (err != 0) + nvme_printf(qpair->ctrlr, + "bus_dmamap_load_bio returned 0x%x!\n", err); + return (err); +} + + + static const char * get_admin_opcode_string(uint16_t opc) { @@ -1090,7 +1133,7 @@ qpair->num_cmds++; } -static void +void nvme_payload_map(void *arg, bus_dma_segment_t *seg, int nseg, int error) { struct nvme_tracker *tr = arg; @@ -1179,44 +1222,7 @@ tr->deadline = SBT_MAX; tr->req = req; - switch (req->type) { - case NVME_REQUEST_VADDR: - KASSERT(req->payload_size <= qpair->ctrlr->max_xfer_size, - ("payload_size (%d) exceeds max_xfer_size (%d)\n", - req->payload_size, qpair->ctrlr->max_xfer_size)); - err = bus_dmamap_load(tr->qpair->dma_tag_payload, - tr->payload_dma_map, req->u.payload, req->payload_size, - nvme_payload_map, tr, 0); - if (err != 0) - nvme_printf(qpair->ctrlr, - "bus_dmamap_load returned 0x%x!\n", err); - break; - case NVME_REQUEST_NULL: - nvme_qpair_submit_tracker(tr->qpair, tr); - break; - case NVME_REQUEST_BIO: - KASSERT(req->u.bio->bio_bcount <= qpair->ctrlr->max_xfer_size, - ("bio->bio_bcount (%jd) exceeds max_xfer_size (%d)\n", - (intmax_t)req->u.bio->bio_bcount, - qpair->ctrlr->max_xfer_size)); - err = bus_dmamap_load_bio(tr->qpair->dma_tag_payload, - tr->payload_dma_map, req->u.bio, nvme_payload_map, tr, 0); - if (err != 0) - nvme_printf(qpair->ctrlr, - "bus_dmamap_load_bio returned 0x%x!\n", err); - break; - case NVME_REQUEST_CCB: - err = bus_dmamap_load_ccb(tr->qpair->dma_tag_payload, - tr->payload_dma_map, req->u.payload, - nvme_payload_map, tr, 0); - if (err != 0) - nvme_printf(qpair->ctrlr, - "bus_dmamap_load_ccb returned 0x%x!\n", err); - break; - default: - panic("unknown nvme request type 0x%x\n", req->type); - break; - } + err = req->payload_dma_map(req, qpair, tr); if (err != 0) { /* diff --git a/sys/dev/nvme/nvme_sim.c b/sys/dev/nvme/nvme_sim.c --- a/sys/dev/nvme/nvme_sim.c +++ b/sys/dev/nvme/nvme_sim.c @@ -62,6 +62,35 @@ struct cam_path *s_path; }; +static int +nvme_req_load_dma_ccb(struct nvme_request *req, struct nvme_qpair *qpair, struct nvme_tracker *tr) +{ + int err; + + err = bus_dmamap_load_ccb(tr->qpair->dma_tag_payload, + tr->payload_dma_map, req->u.payload, + nvme_payload_map, tr, 0); + if (err != 0) + nvme_printf(qpair->ctrlr, + "bus_dmamap_load_ccb returned 0x%x!\n", err); + return (err); +} + +static struct nvme_request * +nvme_allocate_request_ccb(union ccb *ccb, nvme_cb_fn_t cb_fn, void *cb_arg) +{ + struct nvme_request *req; + + req = _nvme_allocate_request(cb_fn, cb_arg); + if (req != NULL) { + req->type = NVME_REQUEST_CCB; + req->u.payload = ccb; + req->payload_dma_map = nvme_req_load_dma_ccb; + } + + return (req); +} + static void nvme_sim_nvmeio_done(void *ccb_arg, const struct nvme_completion *cpl) {