Index: sys/dev/iommu/busdma_iommu.h =================================================================== --- sys/dev/iommu/busdma_iommu.h +++ sys/dev/iommu/busdma_iommu.h @@ -55,6 +55,9 @@ bool locked; bool cansleep; int flags; +#ifdef KMSAN + struct memdesc kmsan_mem; +#endif }; #define BUS_DMAMAP_IOMMU_MALLOC 0x0001 Index: sys/dev/iommu/busdma_iommu.c =================================================================== --- sys/dev/iommu/busdma_iommu.c +++ sys/dev/iommu/busdma_iommu.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -917,11 +918,28 @@ } static void -iommu_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, +iommu_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map1, bus_dmasync_op_t op) { + struct bus_dmamap_iommu *map; + + map = (struct bus_dmamap_iommu *)map1; + kmsan_bus_dmamap_sync(&map->kmsan_mem, op); } +#ifdef KMSAN +static void +iommu_bus_dmamap_load_kmsan(bus_dmamap_t map1, struct memdesc *mem) +{ + struct bus_dmamap_iommu *map; + + map = (struct bus_dmamap_iommu *)map1; + if (map == NULL) + return; + memcpy(&map->kmsan_mem, mem, sizeof(struct memdesc)); +} +#endif + struct bus_dma_impl bus_dma_iommu_impl = { .tag_create = iommu_bus_dma_tag_create, .tag_destroy = iommu_bus_dma_tag_destroy, @@ -938,6 +956,9 @@ .map_complete = iommu_bus_dmamap_complete, .map_unload = iommu_bus_dmamap_unload, .map_sync = iommu_bus_dmamap_sync, +#ifdef KMSAN + .load_kmsan = iommu_bus_dmamap_load_kmsan, +#endif }; static void Index: sys/kern/subr_bus_dma.c =================================================================== --- sys/kern/subr_bus_dma.c +++ sys/kern/subr_bus_dma.c @@ -408,6 +408,11 @@ int error; int nsegs; +#ifdef KMSAN + mem = memdesc_vaddr(buf, buflen); + _bus_dmamap_load_kmsan(dmat, map, &mem); +#endif + if ((flags & BUS_DMA_NOWAIT) == 0) { mem = memdesc_vaddr(buf, buflen); _bus_dmamap_waitok(dmat, map, &mem, callback, callback_arg); @@ -449,6 +454,11 @@ M_ASSERTPKTHDR(m0); +#ifdef KMSAN + struct memdesc mem = memdesc_mbuf(m0); + _bus_dmamap_load_kmsan(dmat, map, &mem); +#endif + flags |= BUS_DMA_NOWAIT; nsegs = -1; error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, NULL, &nsegs, flags); @@ -471,6 +481,11 @@ { int error; +#ifdef KMSAN + struct memdesc mem = memdesc_mbuf(m0); + _bus_dmamap_load_kmsan(dmat, map, &mem); +#endif + flags |= BUS_DMA_NOWAIT; *nsegs = -1; error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, segs, nsegs, flags); @@ -486,6 +501,11 @@ bus_dma_segment_t *segs; int nsegs, error; +#ifdef KMSAN + struct memdesc mem = memdesc_uio(uio); + _bus_dmamap_load_kmsan(dmat, map, &mem); +#endif + flags |= BUS_DMA_NOWAIT; nsegs = -1; error = _bus_dmamap_load_uio(dmat, map, uio, &nsegs, flags); @@ -513,6 +533,11 @@ int error; int nsegs; +#ifdef KMSAN + mem = memdesc_ccb(ccb); + _bus_dmamap_load_kmsan(dmat, map, &mem); +#endif + ccb_h = &ccb->ccb_h; if ((ccb_h->flags & CAM_DIR_MASK) == CAM_DIR_NONE) { callback(callback_arg, NULL, 0, 0); @@ -557,6 +582,11 @@ int error; int nsegs; +#ifdef KMSAN + mem = memdesc_bio(bio); + _bus_dmamap_load_kmsan(dmat, map, &mem); +#endif + if ((flags & BUS_DMA_NOWAIT) == 0) { mem = memdesc_bio(bio); _bus_dmamap_waitok(dmat, map, &mem, callback, callback_arg); @@ -595,6 +625,10 @@ int error; int nsegs; +#ifdef KMSAN + _bus_dmamap_load_kmsan(dmat, map, mem); +#endif + if ((flags & BUS_DMA_NOWAIT) == 0) _bus_dmamap_waitok(dmat, map, mem, callback, callback_arg); Index: sys/x86/include/bus_dma.h =================================================================== --- sys/x86/include/bus_dma.h +++ sys/x86/include/bus_dma.h @@ -191,4 +191,16 @@ return (tc->impl->map_complete(dmat, map, segs, nsegs, error)); } +#ifdef KMSAN +static inline void +_bus_dmamap_load_kmsan(bus_dma_tag_t dmat, bus_dmamap_t map, + struct memdesc *mem) +{ + struct bus_dma_tag_common *tc; + + tc = (struct bus_dma_tag_common *)dmat; + return (tc->impl->load_kmsan(map, mem)); +} +#endif + #endif /* !_X86_BUS_DMA_H_ */ Index: sys/x86/include/busdma_impl.h =================================================================== --- sys/x86/include/busdma_impl.h +++ sys/x86/include/busdma_impl.h @@ -85,6 +85,9 @@ void (*map_unload)(bus_dma_tag_t dmat, bus_dmamap_t map); void (*map_sync)(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op); +#ifdef KMSAN + void (*load_kmsan)(bus_dmamap_t map, struct memdesc *mem); +#endif }; void bus_dma_dflt_lock(void *arg, bus_dma_lock_op_t op); Index: sys/x86/x86/busdma_bounce.c =================================================================== --- sys/x86/x86/busdma_bounce.c +++ sys/x86/x86/busdma_bounce.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -66,6 +67,7 @@ BUS_DMA_COULD_BOUNCE = 0x01, BUS_DMA_MIN_ALLOC_COMP = 0x02, BUS_DMA_KMEM_ALLOC = 0x04, + BUS_DMA_FORCE_MAP = 0x08, }; struct bounce_zone; @@ -128,6 +130,9 @@ bus_dmamap_callback_t *callback; void *callback_arg; STAILQ_ENTRY(bus_dmamap) links; +#ifdef KMSAN + struct memdesc kmsan_mem; +#endif }; static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; @@ -202,6 +207,14 @@ newtag->map_count = 0; newtag->segments = NULL; +#ifdef KMSAN + /* + * When KMSAN is configured, we need a map to store a memory descriptor + * which can be used for validation. + */ + newtag->bounce_flags |= BUS_DMA_FORCE_MAP; +#endif + if (parent != NULL && (newtag->common.filter != NULL || (parent->bounce_flags & BUS_DMA_COULD_BOUNCE) != 0)) newtag->bounce_flags |= BUS_DMA_COULD_BOUNCE; @@ -301,7 +314,7 @@ error = 0; if (dmat->segments == NULL) { - dmat->segments = (bus_dma_segment_t *)malloc_domainset( + dmat->segments = malloc_domainset( sizeof(bus_dma_segment_t) * dmat->common.nsegments, M_DEVBUF, DOMAINSET_PREF(dmat->common.domain), M_NOWAIT); if (dmat->segments == NULL) { @@ -311,6 +324,19 @@ } } + if (dmat->bounce_flags & (BUS_DMA_COULD_BOUNCE | BUS_DMA_FORCE_MAP)) { + *mapp = malloc_domainset(sizeof(**mapp), M_DEVBUF, + DOMAINSET_PREF(dmat->common.domain), M_NOWAIT | M_ZERO); + if (*mapp == NULL) { + CTR3(KTR_BUSDMA, "%s: tag %p error %d", + __func__, dmat, ENOMEM); + return (ENOMEM); + } + STAILQ_INIT(&(*mapp)->bpages); + } else { + *mapp = NULL; + } + /* * Bouncing might be required if the driver asks for an active * exclusion region, a data alignment that is stricter than 1, and/or @@ -318,23 +344,11 @@ */ if ((dmat->bounce_flags & BUS_DMA_COULD_BOUNCE) != 0) { /* Must bounce */ - if (dmat->bounce_zone == NULL) { - if ((error = alloc_bounce_zone(dmat)) != 0) - return (error); - } + if (dmat->bounce_zone == NULL && + (error = alloc_bounce_zone(dmat)) != 0) + goto out; bz = dmat->bounce_zone; - *mapp = (bus_dmamap_t)malloc_domainset(sizeof(**mapp), M_DEVBUF, - DOMAINSET_PREF(dmat->common.domain), M_NOWAIT | M_ZERO); - if (*mapp == NULL) { - CTR3(KTR_BUSDMA, "%s: tag %p error %d", - __func__, dmat, ENOMEM); - return (ENOMEM); - } - - /* Initialize the new map */ - STAILQ_INIT(&((*mapp)->bpages)); - /* * Attempt to add pages to our pool on a per-instance * basis up to a sane limit. @@ -361,11 +375,16 @@ error = 0; } bz->map_count++; + } + +out: + if (error == 0) { + dmat->map_count++; } else { + free(*mapp, M_DEVBUF); *mapp = NULL; } - if (error == 0) - dmat->map_count++; + CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", __func__, dmat, dmat->common.flags, error); return (error); @@ -968,7 +987,10 @@ vm_offset_t datavaddr, tempvaddr; bus_size_t datacount1, datacount2; - if (map == NULL || (bpage = STAILQ_FIRST(&map->bpages)) == NULL) + if (map == NULL) + goto out; + kmsan_bus_dmamap_sync(&map->kmsan_mem, op); + if ((bpage = STAILQ_FIRST(&map->bpages)) == NULL) goto out; /* @@ -1063,6 +1085,16 @@ atomic_thread_fence_rel(); } +#ifdef KMSAN +static void +bounce_bus_dmamap_load_kmsan(bus_dmamap_t map, struct memdesc *mem) +{ + if (map == NULL) + return; + memcpy(&map->kmsan_mem, mem, sizeof(map->kmsan_mem)); +} +#endif + static void init_bounce_pages(void *dummy __unused) { @@ -1344,4 +1376,7 @@ .map_complete = bounce_bus_dmamap_complete, .map_unload = bounce_bus_dmamap_unload, .map_sync = bounce_bus_dmamap_sync, +#ifdef KMSAN + .load_kmsan = bounce_bus_dmamap_load_kmsan, +#endif };