Index: sys/compat/linuxkpi/common/include/linux/dmapool.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/dmapool.h +++ sys/compat/linuxkpi/common/include/linux/dmapool.h @@ -46,15 +46,17 @@ dma_addr_t dma_addr); struct dma_pool { - struct pci_dev *pool_pdev; + struct device *pool_device; uma_zone_t pool_zone; - struct mtx pool_dma_lock; + struct mtx pool_lock; bus_dma_tag_t pool_dmat; size_t pool_entry_size; - struct mtx pool_ptree_lock; struct pctrie pool_ptree; }; +#define DMA_POOL_LOCK(pool) mtx_lock(&(pool)->pool_lock) +#define DMA_POOL_UNLOCK(pool) mtx_unlock(&(pool)->pool_lock) + static inline struct dma_pool * dma_pool_create(char *name, struct device *dev, size_t size, size_t align, size_t boundary) Index: sys/compat/linuxkpi/common/src/linux_pci.c =================================================================== --- sys/compat/linuxkpi/common/src/linux_pci.c +++ sys/compat/linuxkpi/common/src/linux_pci.c @@ -78,11 +78,12 @@ struct linux_dma_priv { uint64_t dma_mask; - struct mtx dma_lock; + struct mtx lock; bus_dma_tag_t dmat; - struct mtx ptree_lock; struct pctrie ptree; }; +#define DMA_PRIV_LOCK(priv) mtx_lock(&(priv)->lock) +#define DMA_PRIV_UNLOCK(priv) mtx_unlock(&(priv)->lock) static int linux_pdev_dma_init(struct pci_dev *pdev) @@ -92,9 +93,8 @@ priv = malloc(sizeof(*priv), M_DEVBUF, M_WAITOK | M_ZERO); pdev->dev.dma_priv = priv; - mtx_init(&priv->dma_lock, "linux_dma", NULL, MTX_DEF); + mtx_init(&priv->lock, "lkpi-priv-dma", NULL, MTX_DEF); - mtx_init(&priv->ptree_lock, "linux_dma_ptree", NULL, MTX_DEF); pctrie_init(&priv->ptree); return (0); @@ -108,8 +108,7 @@ priv = pdev->dev.dma_priv; if (priv->dmat) bus_dma_tag_destroy(priv->dmat); - mtx_destroy(&priv->dma_lock); - mtx_destroy(&priv->ptree_lock); + mtx_destroy(&priv->lock); free(priv, M_DEVBUF); pdev->dev.dma_priv = NULL; return (0); @@ -454,7 +453,6 @@ uma_zfree(linux_dma_trie_zone, node); } - PCTRIE_DEFINE(LINUX_DMA, linux_dma_obj, dma_addr, linux_dma_trie_alloc, linux_dma_trie_free); @@ -493,45 +491,58 @@ { struct linux_dma_priv *priv; struct linux_dma_obj *obj; - int error, nseg; + struct linux_dma_obj *old; bus_dma_segment_t seg; + dma_addr_t retval; + int error; + int nseg; priv = dev->dma_priv; obj = uma_zalloc(linux_dma_obj_zone, 0); - if (bus_dmamap_create(priv->dmat, 0, &obj->dmamap) != 0) { - uma_zfree(linux_dma_obj_zone, obj); - return (0); - } + if (bus_dmamap_create(priv->dmat, 0, &obj->dmamap) != 0) + goto out_error; nseg = -1; - mtx_lock(&priv->dma_lock); - if (_bus_dmamap_load_phys(priv->dmat, obj->dmamap, phys, len, - BUS_DMA_NOWAIT, &seg, &nseg) != 0) { + DMA_PRIV_LOCK(priv); + error = _bus_dmamap_load_phys(priv->dmat, obj->dmamap, phys, len, + BUS_DMA_NOWAIT, &seg, &nseg); + if (error != 0) { bus_dmamap_destroy(priv->dmat, obj->dmamap); - mtx_unlock(&priv->dma_lock); - uma_zfree(linux_dma_obj_zone, obj); - return (0); + DMA_PRIV_UNLOCK(priv); + goto out_error; } - mtx_unlock(&priv->dma_lock); - KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg)); + KASSERT(nseg == 0, ("More than one segment (nseg=%d)", nseg + 1)); obj->dma_addr = seg.ds_addr; - mtx_lock(&priv->ptree_lock); + /* allow new map on same location w/o unmap */ + old = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, obj->dma_addr); + if (old != NULL) { + LINUX_DMA_PCTRIE_REMOVE(&priv->ptree, obj->dma_addr); + bus_dmamap_unload(priv->dmat, old->dmamap); + bus_dmamap_destroy(priv->dmat, old->dmamap); + } + error = LINUX_DMA_PCTRIE_INSERT(&priv->ptree, obj); - mtx_unlock(&priv->ptree_lock); if (error != 0) { - mtx_lock(&priv->dma_lock); bus_dmamap_unload(priv->dmat, obj->dmamap); bus_dmamap_destroy(priv->dmat, obj->dmamap); - mtx_unlock(&priv->dma_lock); - uma_zfree(linux_dma_obj_zone, obj); - return (0); + retval = 0; + } else { + retval = obj->dma_addr; } + DMA_PRIV_UNLOCK(priv); - return (obj->dma_addr); + if (old != NULL) + uma_zfree(linux_dma_obj_zone, old); + + if (error == 0) + return (retval); +out_error: + uma_zfree(linux_dma_obj_zone, obj); + return (0); } void @@ -542,20 +553,15 @@ priv = dev->dma_priv; - mtx_lock(&priv->ptree_lock); + DMA_PRIV_LOCK(priv); obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, dma_addr); - if (obj == NULL) { - mtx_unlock(&priv->ptree_lock); - return; + if (obj != NULL) { + LINUX_DMA_PCTRIE_REMOVE(&priv->ptree, dma_addr); + bus_dmamap_unload(priv->dmat, obj->dmamap); + bus_dmamap_destroy(priv->dmat, obj->dmamap); } - LINUX_DMA_PCTRIE_REMOVE(&priv->ptree, dma_addr); - mtx_unlock(&priv->ptree_lock); + DMA_PRIV_UNLOCK(priv); - mtx_lock(&priv->dma_lock); - bus_dmamap_unload(priv->dmat, obj->dmamap); - bus_dmamap_destroy(priv->dmat, obj->dmamap); - mtx_unlock(&priv->dma_lock); - uma_zfree(linux_dma_obj_zone, obj); } @@ -565,6 +571,7 @@ { struct linux_dma_priv *priv; struct linux_dma_obj *obj; + struct linux_dma_obj *old; struct scatterlist *dma_sg, *sg; int dma_nents, error, nseg; size_t seg_len; @@ -575,14 +582,14 @@ obj = uma_zalloc(linux_dma_obj_zone, 0); - if (bus_dmamap_create(priv->dmat, 0, &obj->dmamap) != 0) { - uma_zfree(linux_dma_obj_zone, obj); - return (0); - } + if (bus_dmamap_create(priv->dmat, 0, &obj->dmamap) != 0) + goto out_error; sg = sgl; dma_sg = sg; dma_nents = 0; + + DMA_PRIV_LOCK(priv); while (nents > 0) { seg_phys = sg_phys(sg); seg_len = sg->length; @@ -595,19 +602,17 @@ } nseg = -1; - mtx_lock(&priv->dma_lock); if (_bus_dmamap_load_phys(priv->dmat, obj->dmamap, seg_phys, seg_len, BUS_DMA_NOWAIT, &seg, &nseg) != 0) { bus_dmamap_unload(priv->dmat, obj->dmamap); bus_dmamap_destroy(priv->dmat, obj->dmamap); - mtx_unlock(&priv->dma_lock); - uma_zfree(linux_dma_obj_zone, obj); - return (0); + DMA_PRIV_UNLOCK(priv); + goto out_error; } - mtx_unlock(&priv->dma_lock); - KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg)); + KASSERT(nseg == 0, ("More than one segment (nseg=%d)", nseg + 1)); + sg_dma_address(dma_sg) = seg.ds_addr; sg_dma_len(dma_sg) = seg.ds_len; @@ -617,19 +622,28 @@ obj->dma_addr = sg_dma_address(sgl); - mtx_lock(&priv->ptree_lock); + old = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, obj->dma_addr); + if (old != NULL) { + LINUX_DMA_PCTRIE_REMOVE(&priv->ptree, obj->dma_addr); + bus_dmamap_unload(priv->dmat, old->dmamap); + bus_dmamap_destroy(priv->dmat, old->dmamap); + } + error = LINUX_DMA_PCTRIE_INSERT(&priv->ptree, obj); - mtx_unlock(&priv->ptree_lock); if (error != 0) { - mtx_lock(&priv->dma_lock); bus_dmamap_unload(priv->dmat, obj->dmamap); bus_dmamap_destroy(priv->dmat, obj->dmamap); - mtx_unlock(&priv->dma_lock); - uma_zfree(linux_dma_obj_zone, obj); - return (0); } + DMA_PRIV_UNLOCK(priv); - return (dma_nents); + if (old != NULL) + uma_zfree(linux_dma_obj_zone, old); + + if (error == 0) + return (dma_nents); +out_error: + uma_zfree(linux_dma_obj_zone, obj); + return (0); } void @@ -641,19 +655,16 @@ priv = dev->dma_priv; - mtx_lock(&priv->ptree_lock); + DMA_PRIV_LOCK(priv); obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, sg_dma_address(sgl)); if (obj == NULL) { - mtx_unlock(&priv->ptree_lock); + DMA_PRIV_UNLOCK(priv); return; } LINUX_DMA_PCTRIE_REMOVE(&priv->ptree, sg_dma_address(sgl)); - mtx_unlock(&priv->ptree_lock); - - mtx_lock(&priv->dma_lock); bus_dmamap_unload(priv->dmat, obj->dmamap); bus_dmamap_destroy(priv->dmat, obj->dmamap); - mtx_unlock(&priv->dma_lock); + DMA_PRIV_UNLOCK(priv); uma_zfree(linux_dma_obj_zone, obj); } @@ -667,15 +678,16 @@ bus_dma_segment_t seg; nseg = -1; - mtx_lock(&pool->pool_dma_lock); + DMA_POOL_LOCK(pool); error = _bus_dmamap_load_phys(pool->pool_dmat, obj->dmamap, vtophys(obj->vaddr), pool->pool_entry_size, BUS_DMA_NOWAIT, &seg, &nseg); - mtx_unlock(&pool->pool_dma_lock); + DMA_POOL_UNLOCK(pool); if (error != 0) { return (error); } - KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg)); + + KASSERT(nseg == 0, ("More than one segment (nseg=%d)", nseg + 1)); obj->dma_addr = seg.ds_addr; return (0); @@ -687,9 +699,9 @@ struct linux_dma_obj *obj = mem; struct dma_pool *pool = arg; - mtx_lock(&pool->pool_dma_lock); + DMA_POOL_LOCK(pool); bus_dmamap_unload(pool->pool_dmat, obj->dmamap); - mtx_unlock(&pool->pool_dma_lock); + DMA_POOL_UNLOCK(pool); } static int @@ -701,7 +713,7 @@ struct linux_dma_obj *obj; int error, i; - priv = pool->pool_pdev->dev.dma_priv; + priv = pool->pool_device->dma_priv; for (i = 0; i < count; i++) { obj = uma_zalloc(linux_dma_obj_zone, flags); if (obj == NULL) @@ -709,7 +721,7 @@ error = bus_dmamem_alloc(pool->pool_dmat, &obj->vaddr, BUS_DMA_NOWAIT, &obj->dmamap); - if (error!= 0) { + if (error != 0) { uma_zfree(linux_dma_obj_zone, obj); break; } @@ -728,7 +740,7 @@ struct linux_dma_obj *obj; int i; - priv = pool->pool_pdev->dev.dma_priv; + priv = pool->pool_device->dma_priv; for (i = 0; i < count; i++) { obj = store[i]; bus_dmamem_free(pool->pool_dmat, obj->vaddr, obj->dmamap); @@ -746,7 +758,7 @@ priv = dev->dma_priv; pool = kzalloc(sizeof(*pool), GFP_KERNEL); - pool->pool_pdev = to_pci_dev(dev); + pool->pool_device = dev; pool->pool_entry_size = size; if (bus_dma_tag_create(bus_get_dma_tag(dev->bsddev), @@ -768,10 +780,7 @@ dma_pool_obj_dtor, NULL, NULL, dma_pool_obj_import, dma_pool_obj_release, pool, 0); - mtx_init(&pool->pool_dma_lock, "linux_dma_pool", NULL, MTX_DEF); - - mtx_init(&pool->pool_ptree_lock, "linux_dma_pool_ptree", NULL, - MTX_DEF); + mtx_init(&pool->pool_lock, "lkpi-dma-pool", NULL, MTX_DEF); pctrie_init(&pool->pool_ptree); return (pool); @@ -783,8 +792,7 @@ uma_zdestroy(pool->pool_zone); bus_dma_tag_destroy(pool->pool_dmat); - mtx_destroy(&pool->pool_ptree_lock); - mtx_destroy(&pool->pool_dma_lock); + mtx_destroy(&pool->pool_lock); kfree(pool); } @@ -798,13 +806,13 @@ if (obj == NULL) return (NULL); - mtx_lock(&pool->pool_ptree_lock); + DMA_POOL_LOCK(pool); if (LINUX_DMA_PCTRIE_INSERT(&pool->pool_ptree, obj) != 0) { - mtx_unlock(&pool->pool_ptree_lock); + DMA_POOL_UNLOCK(pool); uma_zfree_arg(pool->pool_zone, obj, pool); return (NULL); } - mtx_unlock(&pool->pool_ptree_lock); + DMA_POOL_UNLOCK(pool); *handle = obj->dma_addr; return (obj->vaddr); @@ -815,14 +823,14 @@ { struct linux_dma_obj *obj; - mtx_lock(&pool->pool_ptree_lock); + DMA_POOL_LOCK(pool); obj = LINUX_DMA_PCTRIE_LOOKUP(&pool->pool_ptree, dma_addr); if (obj == NULL) { - mtx_unlock(&pool->pool_ptree_lock); + DMA_POOL_UNLOCK(pool); return; } LINUX_DMA_PCTRIE_REMOVE(&pool->pool_ptree, dma_addr); - mtx_unlock(&pool->pool_ptree_lock); + DMA_POOL_UNLOCK(pool); uma_zfree_arg(pool->pool_zone, obj, pool); }