Index: head/sys/dev/mpr/mpr.c =================================================================== --- head/sys/dev/mpr/mpr.c +++ head/sys/dev/mpr/mpr.c @@ -382,7 +382,7 @@ static void mpr_resize_queues(struct mpr_softc *sc) { - u_int reqcr, prireqcr, maxio, sges_per_frame; + u_int reqcr, prireqcr, maxio, sges_per_frame, chain_seg_size; /* * Size the queues. Since the reply queues always need one free @@ -413,15 +413,11 @@ * the size of an IEEE Simple SGE. */ if (sc->facts->MsgVersion >= MPI2_VERSION_02_05) { - sc->chain_seg_size = - htole16(sc->facts->IOCMaxChainSegmentSize); - if (sc->chain_seg_size == 0) { - sc->chain_frame_size = MPR_DEFAULT_CHAIN_SEG_SIZE * - MPR_MAX_CHAIN_ELEMENT_SIZE; - } else { - sc->chain_frame_size = sc->chain_seg_size * - MPR_MAX_CHAIN_ELEMENT_SIZE; - } + chain_seg_size = htole16(sc->facts->IOCMaxChainSegmentSize); + if (chain_seg_size == 0) + chain_seg_size = MPR_DEFAULT_CHAIN_SEG_SIZE; + sc->chain_frame_size = chain_seg_size * + MPR_MAX_CHAIN_ELEMENT_SIZE; } else { sc->chain_frame_size = sc->reqframesz; } @@ -766,11 +762,11 @@ if (sc->queues_dmat != NULL) bus_dma_tag_destroy(sc->queues_dmat); - if (sc->chain_busaddr != 0) + if (sc->chain_frames != NULL) { bus_dmamap_unload(sc->chain_dmat, sc->chain_map); - if (sc->chain_frames != NULL) bus_dmamem_free(sc->chain_dmat, sc->chain_frames, sc->chain_map); + } if (sc->chain_dmat != NULL) bus_dma_tag_destroy(sc->chain_dmat); @@ -1411,11 +1407,36 @@ return (0); } +static void +mpr_load_chains_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error) +{ + struct mpr_softc *sc = arg; + struct mpr_chain *chain; + bus_size_t bo; + int i, o, s; + + if (error != 0) + return; + + for (i = 0, o = 0, s = 0; s < nsegs; s++) { + for (bo = 0; bo + sc->chain_frame_size <= segs[s].ds_len; + bo += sc->chain_frame_size) { + chain = &sc->chains[i++]; + chain->chain =(MPI2_SGE_IO_UNION *)(sc->chain_frames+o); + chain->chain_busaddr = segs[s].ds_addr + bo; + o += sc->chain_frame_size; + mpr_free_chain(sc, chain); + } + if (bo != segs[s].ds_len) + o += segs[s].ds_len - bo; + } + sc->chain_free_lowwater = i; +} + static int mpr_alloc_requests(struct mpr_softc *sc) { struct mpr_command *cm; - struct mpr_chain *chain; int i, rsize, nsegs; rsize = sc->reqframesz * sc->num_reqs; @@ -1444,31 +1465,39 @@ mpr_dprint(sc, MPR_INIT, "request frames busaddr= %#016jx size= %d\n", (uintmax_t)sc->req_busaddr, rsize); + sc->chains = malloc(sizeof(struct mpr_chain) * sc->num_chains, M_MPR, + M_NOWAIT | M_ZERO); + if (!sc->chains) { + mpr_dprint(sc, MPR_ERROR, "Cannot allocate chain memory\n"); + return (ENOMEM); + } rsize = sc->chain_frame_size * sc->num_chains; - if (bus_dma_tag_create( sc->mpr_parent_dmat, /* parent */ + if (bus_dma_tag_create( sc->mpr_parent_dmat, /* parent */ 16, 0, /* algnmnt, boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ - rsize, /* maxsize */ - 1, /* nsegments */ - rsize, /* maxsegsize */ - 0, /* flags */ - NULL, NULL, /* lockfunc, lockarg */ - &sc->chain_dmat)) { + rsize, /* maxsize */ + howmany(rsize, PAGE_SIZE), /* nsegments */ + rsize, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &sc->chain_dmat)) { mpr_dprint(sc, MPR_ERROR, "Cannot allocate chain DMA tag\n"); return (ENOMEM); - } - if (bus_dmamem_alloc(sc->chain_dmat, (void **)&sc->chain_frames, - BUS_DMA_NOWAIT, &sc->chain_map)) { + } + if (bus_dmamem_alloc(sc->chain_dmat, (void **)&sc->chain_frames, + BUS_DMA_NOWAIT | BUS_DMA_ZERO, &sc->chain_map)) { mpr_dprint(sc, MPR_ERROR, "Cannot allocate chain memory\n"); return (ENOMEM); - } - bzero(sc->chain_frames, rsize); - bus_dmamap_load(sc->chain_dmat, sc->chain_map, sc->chain_frames, rsize, - mpr_memaddr_cb, &sc->chain_busaddr, 0); - mpr_dprint(sc, MPR_INIT, "chain frames busaddr= %#016jx size= %d\n", - (uintmax_t)sc->chain_busaddr, rsize); + } + if (bus_dmamap_load(sc->chain_dmat, sc->chain_map, sc->chain_frames, + rsize, mpr_load_chains_cb, sc, BUS_DMA_NOWAIT)) { + mpr_dprint(sc, MPR_ERROR, "Cannot load chain memory\n"); + bus_dmamem_free(sc->chain_dmat, sc->chain_frames, + sc->chain_map); + return (ENOMEM); + } rsize = MPR_SENSE_LEN * sc->num_reqs; if (bus_dma_tag_create( sc->mpr_parent_dmat, /* parent */ @@ -1495,22 +1524,6 @@ mpr_memaddr_cb, &sc->sense_busaddr, 0); mpr_dprint(sc, MPR_INIT, "sense frames busaddr= %#016jx size= %d\n", (uintmax_t)sc->sense_busaddr, rsize); - - sc->chains = malloc(sizeof(struct mpr_chain) * sc->num_chains, M_MPR, - M_WAITOK | M_ZERO); - if (!sc->chains) { - mpr_dprint(sc, MPR_ERROR, "Cannot allocate chain memory\n"); - return (ENOMEM); - } - for (i = 0; i < sc->num_chains; i++) { - chain = &sc->chains[i]; - chain->chain = (MPI2_SGE_IO_UNION *)(sc->chain_frames + - i * sc->chain_frame_size); - chain->chain_busaddr = sc->chain_busaddr + - i * sc->chain_frame_size; - mpr_free_chain(sc, chain); - sc->chain_free_lowwater++; - } /* * Allocate NVMe PRP Pages for NVMe SGL support only if the FW supports Index: head/sys/dev/mpr/mprvar.h =================================================================== --- head/sys/dev/mpr/mprvar.h +++ head/sys/dev/mpr/mprvar.h @@ -41,7 +41,7 @@ #define MPR_PRI_REQ_FRAMES 128 #define MPR_EVT_REPLY_FRAMES 32 #define MPR_REPLY_FRAMES MPR_REQ_FRAMES -#define MPR_CHAIN_FRAMES 2048 +#define MPR_CHAIN_FRAMES 16384 #define MPR_MAXIO_PAGES (-1) #define MPR_SENSE_LEN SSD_FULL_SIZE #define MPR_MSI_MAX 1 @@ -322,7 +322,6 @@ u_int maxio; int chain_free_lowwater; uint32_t chain_frame_size; - uint16_t chain_seg_size; int prp_buffer_size; int prp_pages_free; int prp_pages_free_lowwater; @@ -389,7 +388,6 @@ bus_dmamap_t sense_map; uint8_t *chain_frames; - bus_addr_t chain_busaddr; bus_dma_tag_t chain_dmat; bus_dmamap_t chain_map; Index: head/sys/dev/mps/mps.c =================================================================== --- head/sys/dev/mps/mps.c +++ head/sys/dev/mps/mps.c @@ -743,11 +743,11 @@ if (sc->queues_dmat != NULL) bus_dma_tag_destroy(sc->queues_dmat); - if (sc->chain_busaddr != 0) + if (sc->chain_frames != NULL) { bus_dmamap_unload(sc->chain_dmat, sc->chain_map); - if (sc->chain_frames != NULL) bus_dmamem_free(sc->chain_dmat, sc->chain_frames, sc->chain_map); + } if (sc->chain_dmat != NULL) bus_dma_tag_destroy(sc->chain_dmat); @@ -1370,11 +1370,36 @@ return (0); } +static void +mps_load_chains_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error) +{ + struct mps_softc *sc = arg; + struct mps_chain *chain; + bus_size_t bo; + int i, o, s; + + if (error != 0) + return; + + for (i = 0, o = 0, s = 0; s < nsegs; s++) { + for (bo = 0; bo + sc->reqframesz <= segs[s].ds_len; + bo += sc->reqframesz) { + chain = &sc->chains[i++]; + chain->chain =(MPI2_SGE_IO_UNION *)(sc->chain_frames+o); + chain->chain_busaddr = segs[s].ds_addr + bo; + o += sc->reqframesz; + mps_free_chain(sc, chain); + } + if (bo != segs[s].ds_len) + o += segs[s].ds_len - bo; + } + sc->chain_free_lowwater = i; +} + static int mps_alloc_requests(struct mps_softc *sc) { struct mps_command *cm; - struct mps_chain *chain; int i, rsize, nsegs; rsize = sc->reqframesz * sc->num_reqs; @@ -1403,31 +1428,39 @@ mps_dprint(sc, MPS_INIT, "request frames busaddr= %#016jx size= %d\n", (uintmax_t)sc->req_busaddr, rsize); + sc->chains = malloc(sizeof(struct mps_chain) * sc->num_chains, M_MPT2, + M_NOWAIT | M_ZERO); + if (!sc->chains) { + mps_dprint(sc, MPS_ERROR, "Cannot allocate chain memory\n"); + return (ENOMEM); + } rsize = sc->reqframesz * sc->num_chains; - if (bus_dma_tag_create( sc->mps_parent_dmat, /* parent */ + if (bus_dma_tag_create( sc->mps_parent_dmat, /* parent */ 16, 0, /* algnmnt, boundary */ BUS_SPACE_MAXADDR_32BIT,/* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ - rsize, /* maxsize */ - 1, /* nsegments */ - rsize, /* maxsegsize */ - 0, /* flags */ - NULL, NULL, /* lockfunc, lockarg */ - &sc->chain_dmat)) { + rsize, /* maxsize */ + howmany(rsize, PAGE_SIZE), /* nsegments */ + rsize, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &sc->chain_dmat)) { mps_dprint(sc, MPS_ERROR, "Cannot allocate chain DMA tag\n"); return (ENOMEM); - } - if (bus_dmamem_alloc(sc->chain_dmat, (void **)&sc->chain_frames, - BUS_DMA_NOWAIT, &sc->chain_map)) { + } + if (bus_dmamem_alloc(sc->chain_dmat, (void **)&sc->chain_frames, + BUS_DMA_NOWAIT | BUS_DMA_ZERO, &sc->chain_map)) { mps_dprint(sc, MPS_ERROR, "Cannot allocate chain memory\n"); return (ENOMEM); - } - bzero(sc->chain_frames, rsize); - bus_dmamap_load(sc->chain_dmat, sc->chain_map, sc->chain_frames, rsize, - mps_memaddr_cb, &sc->chain_busaddr, 0); - mps_dprint(sc, MPS_INIT, "chain frames busaddr= %#016jx size= %d\n", - (uintmax_t)sc->chain_busaddr, rsize); + } + if (bus_dmamap_load(sc->chain_dmat, sc->chain_map, sc->chain_frames, + rsize, mps_load_chains_cb, sc, BUS_DMA_NOWAIT)) { + mps_dprint(sc, MPS_ERROR, "Cannot load chain memory\n"); + bus_dmamem_free(sc->chain_dmat, sc->chain_frames, + sc->chain_map); + return (ENOMEM); + } rsize = MPS_SENSE_LEN * sc->num_reqs; if (bus_dma_tag_create( sc->mps_parent_dmat, /* parent */ @@ -1454,22 +1487,6 @@ mps_memaddr_cb, &sc->sense_busaddr, 0); mps_dprint(sc, MPS_INIT, "sense frames busaddr= %#016jx size= %d\n", (uintmax_t)sc->sense_busaddr, rsize); - - sc->chains = malloc(sizeof(struct mps_chain) * sc->num_chains, M_MPT2, - M_WAITOK | M_ZERO); - if(!sc->chains) { - mps_dprint(sc, MPS_ERROR, "Cannot allocate chains memory\n"); - return (ENOMEM); - } - for (i = 0; i < sc->num_chains; i++) { - chain = &sc->chains[i]; - chain->chain = (MPI2_SGE_IO_UNION *)(sc->chain_frames + - i * sc->reqframesz); - chain->chain_busaddr = sc->chain_busaddr + - i * sc->reqframesz; - mps_free_chain(sc, chain); - sc->chain_free_lowwater++; - } nsegs = (sc->maxio / PAGE_SIZE) + 1; if (bus_dma_tag_create( sc->mps_parent_dmat, /* parent */ Index: head/sys/dev/mps/mpsvar.h =================================================================== --- head/sys/dev/mps/mpsvar.h +++ head/sys/dev/mps/mpsvar.h @@ -43,7 +43,7 @@ #define MPS_PRI_REQ_FRAMES 128 #define MPS_EVT_REPLY_FRAMES 32 #define MPS_REPLY_FRAMES MPS_REQ_FRAMES -#define MPS_CHAIN_FRAMES 2048 +#define MPS_CHAIN_FRAMES 16384 #define MPS_MAXIO_PAGES (-1) #define MPS_SENSE_LEN SSD_FULL_SIZE #define MPS_MSI_MAX 1 @@ -378,7 +378,6 @@ bus_dmamap_t sense_map; uint8_t *chain_frames; - bus_addr_t chain_busaddr; bus_dma_tag_t chain_dmat; bus_dmamap_t chain_map;