Index: sys/dev/iommu/busdma_iommu.h =================================================================== --- sys/dev/iommu/busdma_iommu.h +++ sys/dev/iommu/busdma_iommu.h @@ -55,6 +55,9 @@ bool locked; bool cansleep; int flags; +#ifdef KMSAN + struct memdesc kmsan_mem; +#endif }; #define BUS_DMAMAP_IOMMU_MALLOC 0x0001 Index: sys/dev/iommu/busdma_iommu.c =================================================================== --- sys/dev/iommu/busdma_iommu.c +++ sys/dev/iommu/busdma_iommu.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -917,11 +918,28 @@ } static void -iommu_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, +iommu_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map1, bus_dmasync_op_t op) { + struct bus_dmamap_iommu *map; + + map = (struct bus_dmamap_iommu *)map1; + kmsan_bus_dmamap_sync(&map->kmsan_mem, op); } +#ifdef KMSAN +static void +iommu_bus_dmamap_load_kmsan(bus_dmamap_t map1, struct memdesc *mem) +{ + struct bus_dmamap_iommu *map; + + map = (struct bus_dmamap_iommu *)map1; + if (map == NULL) + return; + memcpy(&map->kmsan_mem, mem, sizeof(struct memdesc)); +} +#endif + struct bus_dma_impl bus_dma_iommu_impl = { .tag_create = iommu_bus_dma_tag_create, .tag_destroy = iommu_bus_dma_tag_destroy, @@ -938,6 +956,9 @@ .map_complete = iommu_bus_dmamap_complete, .map_unload = iommu_bus_dmamap_unload, .map_sync = iommu_bus_dmamap_sync, +#ifdef KMSAN + .load_kmsan = iommu_bus_dmamap_load_kmsan, +#endif }; static void Index: sys/kern/subr_bus_dma.c =================================================================== --- sys/kern/subr_bus_dma.c +++ sys/kern/subr_bus_dma.c @@ -408,6 +408,11 @@ int error; int nsegs; +#ifdef KMSAN + mem = memdesc_vaddr(buf, buflen); + _bus_dmamap_load_kmsan(dmat, map, &mem); +#endif + if ((flags & BUS_DMA_NOWAIT) == 0) { mem = memdesc_vaddr(buf, buflen); _bus_dmamap_waitok(dmat, map, &mem, callback, callback_arg); @@ -449,6 +454,11 @@ M_ASSERTPKTHDR(m0); +#ifdef KMSAN + struct memdesc mem = memdesc_mbuf(m0); + _bus_dmamap_load_kmsan(dmat, map, &mem); +#endif + flags |= BUS_DMA_NOWAIT; nsegs = -1; error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, NULL, &nsegs, flags); @@ -471,6 +481,11 @@ { int error; +#ifdef KMSAN + struct memdesc mem = memdesc_mbuf(m0); + _bus_dmamap_load_kmsan(dmat, map, &mem); +#endif + flags |= BUS_DMA_NOWAIT; *nsegs = -1; error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, segs, nsegs, flags); @@ -486,6 +501,11 @@ bus_dma_segment_t *segs; int nsegs, error; +#ifdef KMSAN + struct memdesc mem = memdesc_uio(uio); + _bus_dmamap_load_kmsan(dmat, map, &mem); +#endif + flags |= BUS_DMA_NOWAIT; nsegs = -1; error = _bus_dmamap_load_uio(dmat, map, uio, &nsegs, flags); @@ -513,6 +533,11 @@ int error; int nsegs; +#ifdef KMSAN + mem = memdesc_ccb(ccb); + _bus_dmamap_load_kmsan(dmat, map, &mem); +#endif + ccb_h = &ccb->ccb_h; if ((ccb_h->flags & CAM_DIR_MASK) == CAM_DIR_NONE) { callback(callback_arg, NULL, 0, 0); @@ -557,6 +582,11 @@ int error; int nsegs; +#ifdef KMSAN + mem = memdesc_bio(bio); + _bus_dmamap_load_kmsan(dmat, map, &mem); +#endif + if ((flags & BUS_DMA_NOWAIT) == 0) { mem = memdesc_bio(bio); _bus_dmamap_waitok(dmat, map, &mem, callback, callback_arg); @@ -595,6 +625,10 @@ int error; int nsegs; +#ifdef KMSAN + _bus_dmamap_load_kmsan(dmat, map, mem); +#endif + if ((flags & BUS_DMA_NOWAIT) == 0) _bus_dmamap_waitok(dmat, map, mem, callback, callback_arg); Index: sys/x86/include/bus_dma.h =================================================================== --- sys/x86/include/bus_dma.h +++ sys/x86/include/bus_dma.h @@ -191,4 +191,16 @@ return (tc->impl->map_complete(dmat, map, segs, nsegs, error)); } +#ifdef KMSAN +static inline void +_bus_dmamap_load_kmsan(bus_dma_tag_t dmat, bus_dmamap_t map, + struct memdesc *mem) +{ + struct bus_dma_tag_common *tc; + + tc = (struct bus_dma_tag_common *)dmat; + return (tc->impl->load_kmsan(map, mem)); +} +#endif + #endif /* !_X86_BUS_DMA_H_ */ Index: sys/x86/include/busdma_impl.h =================================================================== --- sys/x86/include/busdma_impl.h +++ sys/x86/include/busdma_impl.h @@ -85,6 +85,9 @@ void (*map_unload)(bus_dma_tag_t dmat, bus_dmamap_t map); void (*map_sync)(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op); +#ifdef KMSAN + void (*load_kmsan)(bus_dmamap_t map, struct memdesc *mem); +#endif }; void bus_dma_dflt_lock(void *arg, bus_dma_lock_op_t op); Index: sys/x86/x86/busdma_bounce.c =================================================================== --- sys/x86/x86/busdma_bounce.c +++ sys/x86/x86/busdma_bounce.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -66,6 +67,7 @@ BUS_DMA_COULD_BOUNCE = 0x01, BUS_DMA_MIN_ALLOC_COMP = 0x02, BUS_DMA_KMEM_ALLOC = 0x04, + BUS_DMA_KMSAN = 0x08, }; struct bounce_zone; @@ -128,6 +130,9 @@ bus_dmamap_callback_t *callback; void *callback_arg; STAILQ_ENTRY(bus_dmamap) links; +#ifdef KMSAN + struct memdesc kmsan_mem; +#endif }; static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; @@ -202,6 +207,10 @@ newtag->map_count = 0; newtag->segments = NULL; +#ifdef KMSAN + newtag->bounce_flags |= BUS_DMA_KMSAN; +#endif + if (parent != NULL && (newtag->common.filter != NULL || (parent->bounce_flags & BUS_DMA_COULD_BOUNCE) != 0)) newtag->bounce_flags |= BUS_DMA_COULD_BOUNCE; @@ -361,6 +370,14 @@ error = 0; } bz->map_count++; + } else if ((dmat->bounce_flags & BUS_DMA_KMSAN) != 0) { + /* + * When KMSAN is configured, we need a map to store a memory + * descriptor which can be used for validation. + */ + *mapp = malloc(sizeof(**mapp), M_DEVBUF, M_NOWAIT | M_ZERO); + if (*mapp == NULL) + return (ENOMEM); } else { *mapp = NULL; } @@ -968,7 +985,10 @@ vm_offset_t datavaddr, tempvaddr; bus_size_t datacount1, datacount2; - if (map == NULL || (bpage = STAILQ_FIRST(&map->bpages)) == NULL) + if (map == NULL) + goto out; + kmsan_bus_dmamap_sync(&map->kmsan_mem, op); + if ((bpage = STAILQ_FIRST(&map->bpages)) == NULL) goto out; /* @@ -1063,6 +1083,16 @@ atomic_thread_fence_rel(); } +#ifdef KMSAN +static void +bounce_bus_dmamap_load_kmsan(bus_dmamap_t map, struct memdesc *mem) +{ + if (map == NULL) + return; + memcpy(&map->kmsan_mem, mem, sizeof(map->kmsan_mem)); +} +#endif + static void init_bounce_pages(void *dummy __unused) { @@ -1344,4 +1374,7 @@ .map_complete = bounce_bus_dmamap_complete, .map_unload = bounce_bus_dmamap_unload, .map_sync = bounce_bus_dmamap_sync, +#ifdef KMSAN + .load_kmsan = bounce_bus_dmamap_load_kmsan, +#endif };