Index: sys/dev/iommu/busdma_iommu.h =================================================================== --- sys/dev/iommu/busdma_iommu.h +++ sys/dev/iommu/busdma_iommu.h @@ -55,6 +55,9 @@ bool locked; bool cansleep; int flags; +#ifdef KMSAN + struct memdesc kmsan_mem; +#endif }; #define BUS_DMAMAP_IOMMU_MALLOC 0x0001 Index: sys/dev/iommu/busdma_iommu.c =================================================================== --- sys/dev/iommu/busdma_iommu.c +++ sys/dev/iommu/busdma_iommu.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -917,11 +918,28 @@ } static void -iommu_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, +iommu_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map1, bus_dmasync_op_t op) { + struct bus_dmamap_iommu *map; + + map = (struct bus_dmamap_iommu *)map1; + kmsan_bus_dmamap_sync(&map->kmsan_mem, op); } +#ifdef KMSAN +static void +iommu_bus_dmamap_load_kmsan(bus_dmamap_t map1, struct memdesc *mem) +{ + struct bus_dmamap_iommu *map; + + map = (struct bus_dmamap_iommu *)map1; + if (map == NULL) + return; + memcpy(&map->kmsan_mem, mem, sizeof(struct memdesc)); +} +#endif + struct bus_dma_impl bus_dma_iommu_impl = { .tag_create = iommu_bus_dma_tag_create, .tag_destroy = iommu_bus_dma_tag_destroy, @@ -938,6 +956,9 @@ .map_complete = iommu_bus_dmamap_complete, .map_unload = iommu_bus_dmamap_unload, .map_sync = iommu_bus_dmamap_sync, +#ifdef KMSAN + .load_kmsan = iommu_bus_dmamap_load_kmsan, +#endif }; static void Index: sys/kern/subr_bus_dma.c =================================================================== --- sys/kern/subr_bus_dma.c +++ sys/kern/subr_bus_dma.c @@ -408,6 +408,11 @@ int error; int nsegs; +#ifdef KMSAN + mem = memdesc_vaddr(buf, buflen); + _bus_dmamap_load_kmsan(dmat, map, &mem); +#endif + if ((flags & BUS_DMA_NOWAIT) == 0) { mem = memdesc_vaddr(buf, buflen); _bus_dmamap_waitok(dmat, map, &mem, callback, callback_arg); @@ -449,6 +454,11 @@ M_ASSERTPKTHDR(m0); +#ifdef KMSAN + struct memdesc mem = memdesc_mbuf(m0); + _bus_dmamap_load_kmsan(dmat, map, &mem); +#endif + flags |= BUS_DMA_NOWAIT; nsegs = -1; error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, NULL, &nsegs, flags); @@ -471,6 +481,11 @@ { int error; +#ifdef KMSAN + struct memdesc mem = memdesc_mbuf(m0); + _bus_dmamap_load_kmsan(dmat, map, &mem); +#endif + flags |= BUS_DMA_NOWAIT; *nsegs = -1; error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, segs, nsegs, flags); @@ -486,6 +501,11 @@ bus_dma_segment_t *segs; int nsegs, error; +#ifdef KMSAN + struct memdesc mem = memdesc_uio(uio); + _bus_dmamap_load_kmsan(dmat, map, &mem); +#endif + flags |= BUS_DMA_NOWAIT; nsegs = -1; error = _bus_dmamap_load_uio(dmat, map, uio, &nsegs, flags); @@ -513,6 +533,11 @@ int error; int nsegs; +#ifdef KMSAN + mem = memdesc_ccb(ccb); + _bus_dmamap_load_kmsan(dmat, map, &mem); +#endif + ccb_h = &ccb->ccb_h; if ((ccb_h->flags & CAM_DIR_MASK) == CAM_DIR_NONE) { callback(callback_arg, NULL, 0, 0); @@ -557,6 +582,11 @@ int error; int nsegs; +#ifdef KMSAN + mem = memdesc_bio(bio); + _bus_dmamap_load_kmsan(dmat, map, &mem); +#endif + if ((flags & BUS_DMA_NOWAIT) == 0) { mem = memdesc_bio(bio); _bus_dmamap_waitok(dmat, map, &mem, callback, callback_arg); @@ -595,6 +625,10 @@ int error; int nsegs; +#ifdef KMSAN + _bus_dmamap_load_kmsan(dmat, map, mem); +#endif + if ((flags & BUS_DMA_NOWAIT) == 0) _bus_dmamap_waitok(dmat, map, mem, callback, callback_arg); Index: sys/x86/include/bus_dma.h =================================================================== --- sys/x86/include/bus_dma.h +++ sys/x86/include/bus_dma.h @@ -191,4 +191,16 @@ return (tc->impl->map_complete(dmat, map, segs, nsegs, error)); } +#ifdef KMSAN +static inline void +_bus_dmamap_load_kmsan(bus_dma_tag_t dmat, bus_dmamap_t map, + struct memdesc *mem) +{ + struct bus_dma_tag_common *tc; + + tc = (struct bus_dma_tag_common *)dmat; + return (tc->impl->load_kmsan(map, mem)); +} +#endif + #endif /* !_X86_BUS_DMA_H_ */ Index: sys/x86/include/busdma_impl.h =================================================================== --- sys/x86/include/busdma_impl.h +++ sys/x86/include/busdma_impl.h @@ -85,6 +85,9 @@ void (*map_unload)(bus_dma_tag_t dmat, bus_dmamap_t map); void (*map_sync)(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op); +#ifdef KMSAN + void (*load_kmsan)(bus_dmamap_t map, struct memdesc *mem); +#endif }; void bus_dma_dflt_lock(void *arg, bus_dma_lock_op_t op); Index: sys/x86/x86/busdma_bounce.c =================================================================== --- sys/x86/x86/busdma_bounce.c +++ sys/x86/x86/busdma_bounce.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -128,6 +129,9 @@ bus_dmamap_callback_t *callback; void *callback_arg; STAILQ_ENTRY(bus_dmamap) links; +#ifdef KMSAN + struct memdesc kmsan_mem; +#endif }; static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; @@ -202,6 +206,14 @@ newtag->map_count = 0; newtag->segments = NULL; +#ifdef KMSAN + /* + * Force the creation of maps. They will be used to store a descriptor + * for loaded memory region(s). + */ + newtag->bounce_flags |= BUS_DMA_COULD_BOUNCE; +#endif + if (parent != NULL && (newtag->common.filter != NULL || (parent->bounce_flags & BUS_DMA_COULD_BOUNCE) != 0)) newtag->bounce_flags |= BUS_DMA_COULD_BOUNCE; @@ -968,7 +980,10 @@ vm_offset_t datavaddr, tempvaddr; bus_size_t datacount1, datacount2; - if (map == NULL || (bpage = STAILQ_FIRST(&map->bpages)) == NULL) + if (map == NULL) + goto out; + kmsan_bus_dmamap_sync(&map->kmsan_mem, op); + if ((bpage = STAILQ_FIRST(&map->bpages)) == NULL) goto out; /* @@ -1063,6 +1078,16 @@ atomic_thread_fence_rel(); } +#ifdef KMSAN +static void +bounce_bus_dmamap_load_kmsan(bus_dmamap_t map, struct memdesc *mem) +{ + if (map == NULL) + return; + memcpy(&map->kmsan_mem, mem, sizeof(map->kmsan_mem)); +} +#endif + static void init_bounce_pages(void *dummy __unused) { @@ -1344,4 +1369,7 @@ .map_complete = bounce_bus_dmamap_complete, .map_unload = bounce_bus_dmamap_unload, .map_sync = bounce_bus_dmamap_sync, +#ifdef KMSAN + .load_kmsan = bounce_bus_dmamap_load_kmsan, +#endif };