diff --git a/sys/compat/linuxkpi/common/include/linux/dma-mapping.h b/sys/compat/linuxkpi/common/include/linux/dma-mapping.h --- a/sys/compat/linuxkpi/common/include/linux/dma-mapping.h +++ b/sys/compat/linuxkpi/common/include/linux/dma-mapping.h @@ -96,13 +96,17 @@ dma_addr_t *dma_handle, gfp_t flag); void *linuxkpi_dmam_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag); -dma_addr_t linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len); -void linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t size); +dma_addr_t linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len); /* backward compat */ +dma_addr_t lkpi_dma_map_phys(struct device *, vm_paddr_t, size_t, + enum dma_data_direction, unsigned long); +void linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t size); /* backward compat */ +void lkpi_dma_unmap(struct device *, dma_addr_t, size_t, + enum dma_data_direction, unsigned long); int linux_dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, - int nents, enum dma_data_direction dir __unused, + int nents, enum dma_data_direction direction, unsigned long attrs __unused); void linux_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, - int nents __unused, enum dma_data_direction dir __unused, + int nents __unused, enum dma_data_direction direction, unsigned long attrs __unused); void linuxkpi_dma_sync(struct device *, dma_addr_t, size_t, bus_dmasync_op_t); @@ -173,16 +177,17 @@ dma_addr_t dma_addr) { - linux_dma_unmap(dev, dma_addr, size); + lkpi_dma_unmap(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0); kmem_free(cpu_addr, size); } static inline dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page, size_t offset, - size_t size, enum dma_data_direction dir, unsigned long attrs) + size_t size, enum dma_data_direction direction, unsigned long attrs) { - return (linux_dma_map_phys(dev, page_to_phys(page) + offset, size)); + return (lkpi_dma_map_phys(dev, page_to_phys(page) + offset, size, + direction, attrs)); } /* linux_dma_(un)map_sg_attrs does not support attrs yet */ @@ -197,7 +202,8 @@ unsigned long offset, size_t size, enum dma_data_direction direction) { - return (linux_dma_map_phys(dev, page_to_phys(page) + offset, size)); + return (lkpi_dma_map_phys(dev, page_to_phys(page) + offset, size, + direction, 0)); } static inline void @@ -205,7 +211,7 @@ enum dma_data_direction direction) { - linux_dma_unmap(dev, dma_address, size); + lkpi_dma_unmap(dev, dma_address, size, direction, 0); } static inline void @@ -263,28 +269,33 @@ linuxkpi_dma_sync(dev, dma, size, op); } +/* (20250329) These four seem to be unused code. */ static inline void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nelems, enum dma_data_direction direction) { + pr_debug("%s:%d: TODO dir %d\n", __func__, __LINE__, direction); } static inline void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nelems, enum dma_data_direction direction) { + pr_debug("%s:%d: TODO dir %d\n", __func__, __LINE__, direction); } static inline void dma_sync_single_range_for_cpu(struct device *dev, dma_addr_t dma_handle, - unsigned long offset, size_t size, int direction) + unsigned long offset, size_t size, enum dma_data_direction direction) { + pr_debug("%s:%d: TODO dir %d\n", __func__, __LINE__, direction); } static inline void dma_sync_single_range_for_device(struct device *dev, dma_addr_t dma_handle, - unsigned long offset, size_t size, int direction) + unsigned long offset, size_t size, enum dma_data_direction direction) { + pr_debug("%s:%d: TODO dir %d\n", __func__, __LINE__, direction); } #define DMA_MAPPING_ERROR (~(dma_addr_t)0) @@ -306,24 +317,17 @@ static inline dma_addr_t _dma_map_single_attrs(struct device *dev, void *ptr, size_t size, - enum dma_data_direction direction, unsigned long attrs __unused) + enum dma_data_direction direction, unsigned long attrs) { - dma_addr_t dma; - - dma = linux_dma_map_phys(dev, vtophys(ptr), size); - if (!dma_mapping_error(dev, dma)) - dma_sync_single_for_device(dev, dma, size, direction); - - return (dma); + return (lkpi_dma_map_phys(dev, vtophys(ptr), size, + direction, attrs)); } static inline void _dma_unmap_single_attrs(struct device *dev, dma_addr_t dma, size_t size, - enum dma_data_direction direction, unsigned long attrs __unused) + enum dma_data_direction direction, unsigned long attrs) { - - dma_sync_single_for_cpu(dev, dma, size, direction); - linux_dma_unmap(dev, dma, size); + lkpi_dma_unmap(dev, dma, size, direction, attrs); } static inline size_t diff --git a/sys/compat/linuxkpi/common/src/linux_pci.c b/sys/compat/linuxkpi/common/src/linux_pci.c --- a/sys/compat/linuxkpi/common/src/linux_pci.c +++ b/sys/compat/linuxkpi/common/src/linux_pci.c @@ -1530,17 +1530,34 @@ #endif dma_addr_t -linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len) +lkpi_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len, + enum dma_data_direction direction, unsigned long attrs) { struct linux_dma_priv *priv; + dma_addr_t dma; priv = dev->dma_priv; - return (linux_dma_map_phys_common(dev, phys, len, priv->dmat)); + dma = linux_dma_map_phys_common(dev, phys, len, priv->dmat); + if (dma_mapping_error(dev, dma)) + return (dma); + + if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) + dma_sync_single_for_device(dev, dma, len, direction); + + return (dma); +} + +/* For backward compat only so we can MFC this. Remove before 15. */ +dma_addr_t +linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len) +{ + return (lkpi_dma_map_phys(dev, phys, len, DMA_NONE, 0)); } #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) void -linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len) +lkpi_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len, + enum dma_data_direction direction, unsigned long attrs) { struct linux_dma_priv *priv; struct linux_dma_obj *obj; @@ -1557,6 +1574,10 @@ return; } LINUX_DMA_PCTRIE_REMOVE(&priv->ptree, dma_addr); + + if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) + dma_sync_single_for_cpu(dev, dma_addr, len, direction); + bus_dmamap_unload(obj->dmat, obj->dmamap); bus_dmamap_destroy(obj->dmat, obj->dmamap); DMA_PRIV_UNLOCK(priv); @@ -1565,11 +1586,19 @@ } #else void -linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len) +lkpi_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len, + enum dma_data_direction direction, unsigned long attrs) { } #endif +/* For backward compat only so we can MFC this. Remove before 15. */ +void +linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len) +{ + lkpi_dma_unmap(dev, dma_addr, len, DMA_NONE, 0); +} + void * linux_dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag) @@ -1671,7 +1700,7 @@ int linux_dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents, - enum dma_data_direction direction, unsigned long attrs __unused) + enum dma_data_direction direction, unsigned long attrs) { struct linux_dma_priv *priv; struct scatterlist *sg; @@ -1705,6 +1734,9 @@ sg_dma_address(sg) = seg.ds_addr; } + if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) != 0) + goto skip_sync; + switch (direction) { case DMA_BIDIRECTIONAL: bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_PREWRITE); @@ -1718,6 +1750,7 @@ default: break; } +skip_sync: DMA_PRIV_UNLOCK(priv); @@ -1727,7 +1760,7 @@ void linux_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents __unused, enum dma_data_direction direction, - unsigned long attrs __unused) + unsigned long attrs) { struct linux_dma_priv *priv; @@ -1735,6 +1768,9 @@ DMA_PRIV_LOCK(priv); + if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) != 0) + goto skip_sync; + switch (direction) { case DMA_BIDIRECTIONAL: bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_POSTREAD); @@ -1749,6 +1785,7 @@ default: break; } +skip_sync: bus_dmamap_unload(priv->dmat, sgl->dma_map); bus_dmamap_destroy(priv->dmat, sgl->dma_map);