Index: dev/acpica/acpi_pcib_acpi.c =================================================================== --- dev/acpica/acpi_pcib_acpi.c +++ dev/acpica/acpi_pcib_acpi.c @@ -60,6 +60,7 @@ struct acpi_hpcib_softc { device_t ap_dev; ACPI_HANDLE ap_handle; + bus_dma_tag_t ap_dma_tag; int ap_flags; uint32_t ap_osc_ctl; @@ -108,6 +109,7 @@ #endif static int acpi_pcib_request_feature(device_t pcib, device_t dev, enum pci_feature feature); +static bus_dma_tag_t acpi_pcib_get_dma_tag(device_t bus, device_t child); static device_method_t acpi_pcib_acpi_methods[] = { /* Device interface */ @@ -136,6 +138,7 @@ DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), DEVMETHOD(bus_get_cpus, acpi_pcib_get_cpus), + DEVMETHOD(bus_get_dma_tag, acpi_pcib_get_dma_tag), /* pcib interface */ DEVMETHOD(pcib_maxslots, pcib_maxslots), @@ -366,6 +369,7 @@ rman_res_t start; int rid; #endif + int error, domain; uint8_t busno; ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__); @@ -537,15 +541,31 @@ acpi_pcib_fetch_prt(dev, &sc->ap_prt); + error = bus_dma_tag_create(bus_get_dma_tag(dev), 1, + PCI_DMA_BOUNDARY, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, + NULL, NULL, BUS_SPACE_MAXSIZE, BUS_SPACE_UNRESTRICTED, + BUS_SPACE_MAXSIZE, 0, NULL, NULL, &sc->ap_dma_tag); + if (error != 0) + goto errout; + error = bus_get_domain(dev, &domain); + if (error == 0) + error = bus_dma_tag_set_domain(sc->ap_dma_tag, domain); + /* Don't fail to attach if the domain can't be queried or set. */ + error = 0; + bus_generic_probe(dev); if (device_add_child(dev, "pci", -1) == NULL) { - device_printf(device_get_parent(dev), "couldn't attach pci bus\n"); -#if defined(NEW_PCIB) && defined(PCI_RES_BUS) - pcib_host_res_free(dev, &sc->ap_host_res); -#endif - return (ENXIO); + error = ENXIO; + goto errout; } return (bus_generic_attach(dev)); + +errout: + device_printf(device_get_parent(dev), "couldn't attach pci bus\n"); +#if defined(NEW_PCIB) && defined(PCI_RES_BUS) + pcib_host_res_free(dev, &sc->ap_host_res); +#endif + return (error); } /* @@ -752,4 +772,14 @@ } return (acpi_pcib_osc(sc, osc_ctl)); +} + +static bus_dma_tag_t +acpi_pcib_get_dma_tag(device_t bus, device_t child) +{ + struct acpi_hpcib_softc *sc; + + sc = device_get_softc(bus); + + return (sc->ap_dma_tag); } Index: kern/kern_malloc.c =================================================================== --- kern/kern_malloc.c +++ kern/kern_malloc.c @@ -95,6 +95,11 @@ dtrace_malloc_probe_func_t dtrace_malloc_probe; #endif +#if defined(INVARIANTS) || defined(MALLOC_MAKE_FAILURES) || \ + defined(DEBUG_MEMGUARD) || defined(DEBUG_REDZONE) +#define MALLOC_DEBUG 1 +#endif + /* * When realloc() is called, if the new size is sufficiently smaller than * the old size, realloc() will allocate a new, smaller block to avoid @@ -416,6 +421,20 @@ return (ret); } +void * +contigmalloc_domain(unsigned long size, struct malloc_type *type, + int domain, int flags, vm_paddr_t low, vm_paddr_t high, + unsigned long alignment, vm_paddr_t boundary) +{ + void *ret; + + ret = (void *)kmem_alloc_contig_domain(domain, size, flags, low, high, + alignment, boundary, VM_MEMATTR_DEFAULT); + if (ret != NULL) + malloc_type_allocated(type, round_page(size)); + return (ret); +} + /* * contigfree: * @@ -431,26 +450,14 @@ malloc_type_freed(type, round_page(size)); } -/* - * malloc: - * - * Allocate a block of memory. - * - * If M_NOWAIT is set, this routine will not block and return NULL if - * the allocation fails. - */ -void * -malloc(unsigned long size, struct malloc_type *mtp, int flags) +#ifdef MALLOC_DEBUG +static int +malloc_dbg(caddr_t *vap, unsigned long *sizep, struct malloc_type *mtp, + int flags) { +#ifdef INVARIANTS int indx; - struct malloc_type_internal *mtip; - caddr_t va; - uma_zone_t zone; -#if defined(DIAGNOSTIC) || defined(DEBUG_REDZONE) - unsigned long osize = size; -#endif -#ifdef INVARIANTS KASSERT(mtp->ks_magic == M_MAGIC, ("malloc: bad malloc type magic")); /* * Check that exactly one of M_WAITOK or M_NOWAIT is specified. @@ -473,7 +480,8 @@ if ((malloc_nowait_count % malloc_failure_rate) == 0) { atomic_add_int(&malloc_failure_count, 1); t_malloc_fail = time_uptime; - return (NULL); + *vap = NULL; + return (EJUSTRETURN); } } #endif @@ -486,16 +494,46 @@ #ifdef DEBUG_MEMGUARD if (memguard_cmp_mtp(mtp, size)) { va = memguard_alloc(size, flags); - if (va != NULL) - return (va); + if (va != NULL) { + *vap = va; + return (EJUSTRETURN); + } /* This is unfortunate but should not be fatal. */ } #endif #ifdef DEBUG_REDZONE - size = redzone_size_ntor(size); + *sizep = redzone_size_ntor(*sizep); #endif + return (0); +} +#endif + +/* + * malloc: + * + * Allocate a block of memory. + * + * If M_NOWAIT is set, this routine will not block and return NULL if + * the allocation fails. + */ +void * +malloc(unsigned long size, struct malloc_type *mtp, int flags) +{ + int indx; + struct malloc_type_internal *mtip; + caddr_t va; + uma_zone_t zone; +#if defined(DIAGNOSTIC) || defined(DEBUG_REDZONE) + unsigned long osize = size; +#endif + +#ifdef MALLOC_DEBUG + if (malloc_dbg(&va, &size, mtp, flags) != 0) + return (va); +#endif + if (size <= kmem_zmax) { mtip = mtp->ks_handle; if (size & KMEM_ZMASK) @@ -522,11 +560,55 @@ KASSERT(va != NULL, ("malloc(M_WAITOK) returned NULL")); else if (va == NULL) t_malloc_fail = time_uptime; -#ifdef DIAGNOSTIC - if (va != NULL && !(flags & M_ZERO)) { - memset(va, 0x70, osize); - } +#ifdef DEBUG_REDZONE + if (va != NULL) + va = redzone_setup(va, osize); #endif + return ((void *) va); +} + +void * +malloc_domain(unsigned long size, struct malloc_type *mtp, int domain, + int flags) +{ + int indx; + struct malloc_type_internal *mtip; + caddr_t va; + uma_zone_t zone; +#if defined(DIAGNOSTIC) || defined(DEBUG_REDZONE) + unsigned long osize = size; +#endif + +#ifdef MALLOC_DEBUG + if (malloc_dbg(&va, &size, mtp, flags) != 0) + return (va); +#endif + if (size <= kmem_zmax) { + mtip = mtp->ks_handle; + if (size & KMEM_ZMASK) + size = (size & ~KMEM_ZMASK) + KMEM_ZBASE; + indx = kmemsize[size >> KMEM_ZSHIFT]; + KASSERT(mtip->mti_zone < numzones, + ("mti_zone %u out of range %d", + mtip->mti_zone, numzones)); + zone = kmemzones[indx].kz_zone[mtip->mti_zone]; +#ifdef MALLOC_PROFILE + krequests[size >> KMEM_ZSHIFT]++; +#endif + va = uma_zalloc_domain(zone, NULL, domain, flags); + if (va != NULL) + size = zone->uz_size; + malloc_type_zone_allocated(mtp, va == NULL ? 0 : size, indx); + } else { + size = roundup(size, PAGE_SIZE); + zone = NULL; + va = uma_large_malloc_domain(size, domain, flags); + malloc_type_allocated(mtp, va == NULL ? 0 : size); + } + if (flags & M_WAITOK) + KASSERT(va != NULL, ("malloc(M_WAITOK) returned NULL")); + else if (va == NULL) + t_malloc_fail = time_uptime; #ifdef DEBUG_REDZONE if (va != NULL) va = redzone_setup(va, osize); @@ -534,66 +616,124 @@ return ((void *) va); } -/* - * free: - * - * Free a block of memory allocated by malloc. - * - * This routine may not block. - */ -void -free(void *addr, struct malloc_type *mtp) +#ifdef INVARIANTS +static void +free_save_type(void *addr, struct malloc_type *mtp, u_long size) { - uma_slab_t slab; - u_long size; + struct malloc_type **mtpp = addr; + /* + * Cache a pointer to the malloc_type that most recently freed + * this memory here. This way we know who is most likely to + * have stepped on it later. + * + * This code assumes that size is a multiple of 8 bytes for + * 64 bit machines + */ + mtpp = (struct malloc_type **) ((unsigned long)mtpp & ~UMA_ALIGN_PTR); + mtpp += (size - sizeof(struct malloc_type *)) / + sizeof(struct malloc_type *); + *mtpp = mtp; +} +#endif + +#ifdef MALLOC_DEBUG +static int +free_dbg(void **addrp, struct malloc_type *mtp) +{ + void *addr; + + addr = *addrp; KASSERT(mtp->ks_magic == M_MAGIC, ("free: bad malloc type magic")); KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), ("free: called with spinlock or critical section held")); /* free(NULL, ...) does nothing */ if (addr == NULL) - return; + return (EJUSTRETURN); #ifdef DEBUG_MEMGUARD if (is_memguard_addr(addr)) { memguard_free(addr); - return; + return (EJUSTRETURN); } #endif #ifdef DEBUG_REDZONE redzone_check(addr); - addr = redzone_addr_ntor(addr); + *addrp = redzone_addr_ntor(addr); #endif - slab = vtoslab((vm_offset_t)addr & (~UMA_SLAB_MASK)); + return (0); +} +#endif +/* + * free: + * + * Free a block of memory allocated by malloc. + * + * This routine may not block. + */ +void +free(void *addr, struct malloc_type *mtp) +{ + uma_slab_t slab; + u_long size; + +#ifdef MALLOC_DEBUG + if (free_dbg(&addr, mtp) != 0) + return; +#endif + /* free(NULL, ...) does nothing */ + if (addr == NULL) + return; + + slab = vtoslab((vm_offset_t)addr & (~UMA_SLAB_MASK)); if (slab == NULL) panic("free: address %p(%p) has not been allocated.\n", addr, (void *)((u_long)addr & (~UMA_SLAB_MASK))); if (!(slab->us_flags & UMA_SLAB_MALLOC)) { + size = slab->us_keg->uk_size; #ifdef INVARIANTS - struct malloc_type **mtpp = addr; + free_save_type(addr, mtp, size); #endif + uma_zfree_arg(LIST_FIRST(&slab->us_keg->uk_zones), addr, slab); + } else { + size = slab->us_size; + uma_large_free(slab); + } + malloc_type_freed(mtp, size); +} + +void +free_domain(void *addr, struct malloc_type *mtp) +{ + uma_slab_t slab; + u_long size; + +#ifdef MALLOC_DEBUG + if (free_dbg(&addr, mtp) != 0) + return; +#endif + + /* free(NULL, ...) does nothing */ + if (addr == NULL) + return; + + slab = vtoslab((vm_offset_t)addr & (~UMA_SLAB_MASK)); + if (slab == NULL) + panic("free_domain: address %p(%p) has not been allocated.\n", + addr, (void *)((u_long)addr & (~UMA_SLAB_MASK))); + + if (!(slab->us_flags & UMA_SLAB_MALLOC)) { size = slab->us_keg->uk_size; #ifdef INVARIANTS - /* - * Cache a pointer to the malloc_type that most recently freed - * this memory here. This way we know who is most likely to - * have stepped on it later. - * - * This code assumes that size is a multiple of 8 bytes for - * 64 bit machines - */ - mtpp = (struct malloc_type **) - ((unsigned long)mtpp & ~UMA_ALIGN_PTR); - mtpp += (size - sizeof(struct malloc_type *)) / - sizeof(struct malloc_type *); - *mtpp = mtp; + free_save_type(addr, mtp, size); #endif - uma_zfree_arg(LIST_FIRST(&slab->us_keg->uk_zones), addr, slab); + uma_zfree_domain(LIST_FIRST(&slab->us_keg->uk_zones), + addr, slab); } else { size = slab->us_size; uma_large_free(slab); Index: sys/bus_dma.h =================================================================== --- sys/bus_dma.h +++ sys/bus_dma.h @@ -176,6 +176,14 @@ bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, void *lockfuncarg, bus_dma_tag_t *dmat); +/* + * Set the memory domain to be used for allocations. + * + * Automatic for PCI devices. Must be set prior to creating maps or + * allocating memory. + */ +int bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain); + int bus_dma_tag_destroy(bus_dma_tag_t dmat); /* Index: sys/malloc.h =================================================================== --- sys/malloc.h +++ sys/malloc.h @@ -174,8 +174,16 @@ vm_paddr_t low, vm_paddr_t high, unsigned long alignment, vm_paddr_t boundary) __malloc_like __result_use_check __alloc_size(1) __alloc_align(6); +void *contigmalloc_domain(unsigned long size, struct malloc_type *type, + int domain, int flags, vm_paddr_t low, vm_paddr_t high, + unsigned long alignment, vm_paddr_t boundary) + __malloc_like __result_use_check __alloc_size(1) __alloc_align(6); void free(void *addr, struct malloc_type *type); +void free_domain(void *addr, struct malloc_type *type); void *malloc(unsigned long size, struct malloc_type *type, int flags) + __malloc_like __result_use_check __alloc_size(1); +void *malloc_domain(unsigned long size, struct malloc_type *type, + int domain, int flags) __malloc_like __result_use_check __alloc_size(1); void malloc_init(void *); int malloc_last_fail(void); Index: vm/uma.h =================================================================== --- vm/uma.h +++ vm/uma.h @@ -332,6 +332,19 @@ void *uma_zalloc_arg(uma_zone_t zone, void *arg, int flags); /* + * Allocate an item from a specific NUMA domain. This uses a slow path in + * the allocator but is guaranteed to allocate memory from the requested + * domain if M_WAITOK is set. + * + * Arguments: + * zone The zone we are allocating from + * arg This data is passed to the ctor function + * domain The domain to allocate from. + * flags See sys/malloc.h for available flags. + */ +void *uma_zalloc_domain(uma_zone_t zone, void *arg, int domain, int flags); + +/* * Allocates an item out of a zone without supplying an argument * * This is just a wrapper for uma_zalloc_arg for convenience. @@ -358,6 +371,16 @@ */ void uma_zfree_arg(uma_zone_t zone, void *item, void *arg); + +/* + * Frees an item back to the specified zone's domain specific pool. + * + * Arguments: + * zone The zone the item was originally allocated out of. + * item The memory to be freed. + * arg Argument passed to the destructor + */ +void uma_zfree_domain(uma_zone_t zone, void *item, void *arg); /* * Frees an item back to a zone without supplying an argument Index: vm/uma_core.c =================================================================== --- vm/uma_core.c +++ vm/uma_core.c @@ -221,6 +221,8 @@ */ enum zfreeskip { SKIP_NONE = 0, SKIP_DTOR, SKIP_FINI }; +#define UMA_ANYDOMAIN -1 /* Special value for domain search. */ + /* Prototypes.. */ static void *noobj_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); @@ -564,8 +566,8 @@ M_UMAHASH, M_NOWAIT); } else { alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT; - hash->uh_slab_hash = zone_alloc_item(hashzone, NULL, 0, - M_WAITOK); + hash->uh_slab_hash = zone_alloc_item(hashzone, NULL, + UMA_ANYDOMAIN, M_WAITOK); hash->uh_hashsize = UMA_HASH_SIZE_INIT; } if (hash->uh_slab_hash) { @@ -1878,7 +1880,7 @@ args.align = (align == UMA_ALIGN_CACHE) ? uma_align_cache : align; args.flags = flags; args.zone = zone; - return (zone_alloc_item(kegs, &args, 0, M_WAITOK)); + return (zone_alloc_item(kegs, &args, UMA_ANYDOMAIN, M_WAITOK)); } /* See uma.h */ @@ -1935,7 +1937,7 @@ sx_slock(&uma_drain_lock); locked = true; } - res = zone_alloc_item(zones, &args, 0, M_WAITOK); + res = zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK); if (locked) sx_sunlock(&uma_drain_lock); return (res); @@ -1970,7 +1972,7 @@ locked = true; } /* XXX Attaches only one keg of potentially many. */ - res = zone_alloc_item(zones, &args, 0, M_WAITOK); + res = zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK); if (locked) sx_sunlock(&uma_drain_lock); return (res); @@ -1997,7 +1999,7 @@ args.align = 0; args.flags = flags; - return (zone_alloc_item(zones, &args, 0, M_WAITOK)); + return (zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK)); } static void @@ -2206,7 +2208,7 @@ if (zone->uz_flags & UMA_ZONE_NUMA) domain = PCPU_GET(domain); else - domain = 0; + domain = UMA_ANYDOMAIN; /* Short-circuit for zones without buckets and low memory. */ if (zone->uz_count == 0 || bucketdisable) @@ -2248,7 +2250,10 @@ /* * Check the zone's cache of buckets. */ - zdom = &zone->uz_domain[domain]; + if (domain == UMA_ANYDOMAIN) + zdom = &zone->uz_domain[0]; + else + zdom = &zone->uz_domain[domain]; if ((bucket = LIST_FIRST(&zdom->uzd_buckets)) != NULL) { KASSERT(bucket->ub_cnt != 0, ("uma_zalloc_arg: Returning an empty bucket.")); @@ -2306,6 +2311,28 @@ return (item); } +void * +uma_zalloc_domain(uma_zone_t zone, void *udata, int domain, int flags) +{ + + /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ + random_harvest_fast_uma(&zone, sizeof(zone), 1, RANDOM_UMA); + + /* This is the fast path allocation */ + CTR5(KTR_UMA, + "uma_zalloc_domain thread %x zone %s(%p) domain %d flags %d", + curthread, zone->uz_name, zone, domain, flags); + + if (flags & M_WAITOK) { + WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, + "uma_zalloc_domain: zone \"%s\"", zone->uz_name); + } + KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), + ("uma_zalloc_domain: called with spinlock or critical section held")); + + return (zone_alloc_item(zone, udata, domain, flags)); +} + /* * Find a slab with some space. Prefer slabs that are partially used over those * that are totally full. This helps to reduce fragmentation. @@ -2360,7 +2387,9 @@ * Round-robin for non first-touch zones when there is more than one * domain. */ - rr = (zone->uz_flags & UMA_ZONE_NUMA) == 0 && vm_ndomains != 1; + if (vm_ndomains == 1) + rdomain = 0; + rr = rdomain == UMA_ANYDOMAIN; if (rr) { keg->uk_cursor = (keg->uk_cursor + 1) % vm_ndomains; domain = start = keg->uk_cursor; @@ -2665,6 +2694,7 @@ * Arguments * zone The zone to alloc for. * udata The data to be passed to the constructor. + * domain The domain to allocate from or UMA_ANYDOMAIN. * flags M_WAITOK, M_NOWAIT, M_ZERO. * * Returns @@ -2896,6 +2926,25 @@ return; } +void +uma_zfree_domain(uma_zone_t zone, void *item, void *udata) +{ + + /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ + random_harvest_fast_uma(&zone, sizeof(zone), 1, RANDOM_UMA); + + CTR2(KTR_UMA, "uma_zfree_domain thread %x zone %s", curthread, + zone->uz_name); + + KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), + ("uma_zfree_domain: called with spinlock or critical section held")); + + /* uma_zfree(..., NULL) does nothing, to match free(9). */ + if (item == NULL) + return; + zone_free_item(zone, item, udata, SKIP_NONE); +} + static void slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item) { @@ -3342,15 +3391,18 @@ } void * -uma_large_malloc(vm_size_t size, int wait) +uma_large_malloc_domain(vm_size_t size, int domain, int wait) { vm_offset_t addr; uma_slab_t slab; - slab = zone_alloc_item(slabzone, NULL, 0, wait); + slab = zone_alloc_item(slabzone, NULL, domain, wait); if (slab == NULL) return (NULL); - addr = kmem_malloc(kernel_arena, size, wait); + if (domain == UMA_ANYDOMAIN) + addr = kmem_malloc(kernel_arena, size, wait); + else + addr = kmem_malloc_domain(domain, size, wait); if (addr != 0) { vsetslab(addr, slab); slab->us_data = (void *)addr; @@ -3364,6 +3416,13 @@ } return ((void *)addr); +} + +void * +uma_large_malloc(vm_size_t size, int wait) +{ + + return uma_large_malloc_domain(size, UMA_ANYDOMAIN, wait); } void Index: vm/uma_int.h =================================================================== --- vm/uma_int.h +++ vm/uma_int.h @@ -386,6 +386,7 @@ /* Internal prototypes */ static __inline uma_slab_t hash_sfind(struct uma_hash *hash, uint8_t *data); void *uma_large_malloc(vm_size_t size, int wait); +void *uma_large_malloc_domain(vm_size_t size, int domain, int wait); void uma_large_free(uma_slab_t slab); /* Lock Macros */ Index: vm/vm_phys.h =================================================================== --- vm/vm_phys.h +++ vm/vm_phys.h @@ -77,7 +77,7 @@ vm_page_t vm_phys_alloc_freelist_pages(int domain, int freelist, int pool, int order); vm_page_t vm_phys_alloc_pages(int domain, int pool, int order); -boolean_t vm_phys_domain_intersects(long mask, vm_paddr_t low, vm_paddr_t high); +int vm_phys_domain_match(int prefer, vm_paddr_t low, vm_paddr_t high); int vm_phys_fictitious_reg_range(vm_paddr_t start, vm_paddr_t end, vm_memattr_t memattr); void vm_phys_fictitious_unreg_range(vm_paddr_t start, vm_paddr_t end); Index: vm/vm_phys.c =================================================================== --- vm/vm_phys.c +++ vm/vm_phys.c @@ -198,20 +198,25 @@ (uintmax_t)p1->end, (uintmax_t)p2->start, (uintmax_t)p2->end); } -boolean_t -vm_phys_domain_intersects(long mask, vm_paddr_t low, vm_paddr_t high) +int +vm_phys_domain_match(int prefer, vm_paddr_t low, vm_paddr_t high) { - struct vm_phys_seg *s; - int idx; + domainset_t mask; + int i; - while ((idx = ffsl(mask)) != 0) { - idx--; /* ffsl counts from 1 */ - mask &= ~(1UL << idx); - s = &vm_phys_segs[idx]; - if (low < s->end && high > s->start) - return (TRUE); - } - return (FALSE); + DOMAINSET_ZERO(&mask); + /* + * Check for any memory that overlaps low, high. + */ + for (i = 0; i < vm_ndomains; i++) + if (mem_affinity[i].start <= high && + mem_affinity[i].end >= low) + DOMAINSET_SET(mem_affinity[i].domain, &mask); + if (prefer != -1 && DOMAINSET_ISSET(prefer, &mask)) + return (prefer); + if (DOMAINSET_EMPTY(&mask)) + panic("vm_phys_domain_match: Impossible constraint"); + return (DOMAINSET_FFS(&mask)); } /* Index: x86/include/busdma_impl.h =================================================================== --- x86/include/busdma_impl.h +++ x86/include/busdma_impl.h @@ -50,6 +50,7 @@ bus_dma_lock_t *lockfunc; void *lockfuncarg; int ref_count; + int domain; }; struct bus_dma_impl { @@ -60,6 +61,7 @@ bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, void *lockfuncarg, bus_dma_tag_t *dmat); int (*tag_destroy)(bus_dma_tag_t dmat); + int (*tag_set_domain)(bus_dma_tag_t); int (*map_create)(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp); int (*map_destroy)(bus_dma_tag_t dmat, bus_dmamap_t map); int (*mem_alloc)(bus_dma_tag_t dmat, void** vaddr, int flags, Index: x86/iommu/busdma_dmar.c =================================================================== --- x86/iommu/busdma_dmar.c +++ x86/iommu/busdma_dmar.c @@ -326,6 +326,13 @@ } static int +dmar_bus_dma_tag_set_domain(bus_dma_tag_t dmat) +{ + + return (0); +} + +static int dmar_bus_dma_tag_destroy(bus_dma_tag_t dmat1) { struct bus_dma_tag_dmar *dmat, *dmat_copy, *parent; @@ -345,7 +352,7 @@ 1) { if (dmat == &dmat->ctx->ctx_tag) dmar_free_ctx(dmat->ctx); - free(dmat->segments, M_DMAR_DMAMAP); + free_domain(dmat->segments, M_DMAR_DMAMAP); free(dmat, M_DEVBUF); dmat = parent; } else @@ -366,16 +373,18 @@ WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "%s", __func__); tag = (struct bus_dma_tag_dmar *)dmat; - map = malloc(sizeof(*map), M_DMAR_DMAMAP, M_NOWAIT | M_ZERO); + map = malloc_domain(sizeof(*map), M_DMAR_DMAMAP, + tag->common.domain, M_NOWAIT | M_ZERO); if (map == NULL) { *mapp = NULL; return (ENOMEM); } if (tag->segments == NULL) { - tag->segments = malloc(sizeof(bus_dma_segment_t) * - tag->common.nsegments, M_DMAR_DMAMAP, M_NOWAIT); + tag->segments = malloc_domain(sizeof(bus_dma_segment_t) * + tag->common.nsegments, M_DMAR_DMAMAP, + tag->common.domain, M_NOWAIT); if (tag->segments == NULL) { - free(map, M_DMAR_DMAMAP); + free_domain(map, M_DMAR_DMAMAP); *mapp = NULL; return (ENOMEM); } @@ -407,7 +416,7 @@ return (EBUSY); } DMAR_DOMAIN_UNLOCK(domain); - free(map, M_DMAR_DMAMAP); + free_domain(map, M_DMAR_DMAMAP); } tag->map_count--; return (0); @@ -438,10 +447,11 @@ if (tag->common.maxsize < PAGE_SIZE && tag->common.alignment <= tag->common.maxsize && attr == VM_MEMATTR_DEFAULT) { - *vaddr = malloc(tag->common.maxsize, M_DEVBUF, mflags); + *vaddr = malloc_domain(tag->common.maxsize, M_DEVBUF, + tag->common.domain, mflags); map->flags |= BUS_DMAMAP_DMAR_MALLOC; } else { - *vaddr = (void *)kmem_alloc_attr(kernel_arena, + *vaddr = (void *)kmem_alloc_attr_domain(tag->common.domain, tag->common.maxsize, mflags, 0ul, BUS_SPACE_MAXADDR, attr); map->flags |= BUS_DMAMAP_DMAR_KMEM_ALLOC; @@ -464,7 +474,7 @@ map = (struct bus_dmamap_dmar *)map1; if ((map->flags & BUS_DMAMAP_DMAR_MALLOC) != 0) { - free(vaddr, M_DEVBUF); + free_domain(vaddr, M_DEVBUF); map->flags &= ~BUS_DMAMAP_DMAR_MALLOC; } else { KASSERT((map->flags & BUS_DMAMAP_DMAR_KMEM_ALLOC) != 0, @@ -832,6 +842,7 @@ struct bus_dma_impl bus_dma_dmar_impl = { .tag_create = dmar_bus_dma_tag_create, .tag_destroy = dmar_bus_dma_tag_destroy, + .tag_set_domain = dmar_bus_dma_tag_set_domain, .map_create = dmar_bus_dmamap_create, .map_destroy = dmar_bus_dmamap_destroy, .mem_alloc = dmar_bus_dmamem_alloc, @@ -842,7 +853,7 @@ .map_waitok = dmar_bus_dmamap_waitok, .map_complete = dmar_bus_dmamap_complete, .map_unload = dmar_bus_dmamap_unload, - .map_sync = dmar_bus_dmamap_sync + .map_sync = dmar_bus_dmamap_sync, }; static void Index: x86/x86/busdma_bounce.c =================================================================== --- x86/x86/busdma_bounce.c +++ x86/x86/busdma_bounce.c @@ -99,6 +99,7 @@ int total_bounced; int total_deferred; int map_count; + int domain; bus_size_t alignment; bus_addr_t lowaddr; char zoneid[8]; @@ -150,6 +151,32 @@ static int _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags); +static int +bounce_bus_dma_zone_setup(bus_dma_tag_t dmat) +{ + struct bounce_zone *bz; + int error; + + /* Must bounce */ + if ((error = alloc_bounce_zone(dmat)) != 0) + return (error); + bz = dmat->bounce_zone; + + if (ptoa(bz->total_bpages) < dmat->common.maxsize) { + int pages; + + pages = atop(dmat->common.maxsize) - bz->total_bpages; + + /* Add pages to our bounce pool */ + if (alloc_bounce_pages(dmat, pages) < pages) + return (ENOMEM); + } + /* Performed initial allocation */ + dmat->bounce_flags |= BUS_DMA_MIN_ALLOC_COMP; + + return (0); +} + /* * Allocate a device specific dma_tag. */ @@ -184,28 +211,9 @@ newtag->bounce_flags |= BUS_DMA_COULD_BOUNCE; if (((newtag->bounce_flags & BUS_DMA_COULD_BOUNCE) != 0) && - (flags & BUS_DMA_ALLOCNOW) != 0) { - struct bounce_zone *bz; - - /* Must bounce */ - if ((error = alloc_bounce_zone(newtag)) != 0) { - free(newtag, M_DEVBUF); - return (error); - } - bz = newtag->bounce_zone; - - if (ptoa(bz->total_bpages) < maxsize) { - int pages; - - pages = atop(maxsize) - bz->total_bpages; - - /* Add pages to our bounce pool */ - if (alloc_bounce_pages(newtag, pages) < pages) - error = ENOMEM; - } - /* Performed initial allocation */ - newtag->bounce_flags |= BUS_DMA_MIN_ALLOC_COMP; - } else + (flags & BUS_DMA_ALLOCNOW) != 0) + error = bounce_bus_dma_zone_setup(newtag); + else error = 0; if (error != 0) @@ -218,7 +226,24 @@ return (error); } +/* + * Update the domain for the tag. We may need to reallocate the zone and + * bounce pages. + */ static int +bounce_bus_dma_tag_set_domain(bus_dma_tag_t dmat) +{ + + KASSERT(dmat->map_count == 0, + ("bounce_bus_dma_tag_set_domain: Domain set after use.\n")); + if ((dmat->bounce_flags & BUS_DMA_COULD_BOUNCE) == 0 || + dmat->bounce_zone == NULL) + return (0); + dmat->bounce_flags &= ~BUS_DMA_MIN_ALLOC_COMP; + return (bounce_bus_dma_zone_setup(dmat)); +} + +static int bounce_bus_dma_tag_destroy(bus_dma_tag_t dmat) { bus_dma_tag_t dmat_copy, parent; @@ -237,7 +262,7 @@ atomic_subtract_int(&dmat->common.ref_count, 1); if (dmat->common.ref_count == 0) { if (dmat->segments != NULL) - free(dmat->segments, M_DEVBUF); + free_domain(dmat->segments, M_DEVBUF); free(dmat, M_DEVBUF); /* * Last reference count, so @@ -269,9 +294,9 @@ error = 0; if (dmat->segments == NULL) { - dmat->segments = (bus_dma_segment_t *)malloc( + dmat->segments = (bus_dma_segment_t *)malloc_domain( sizeof(bus_dma_segment_t) * dmat->common.nsegments, - M_DEVBUF, M_NOWAIT); + M_DEVBUF, dmat->common.domain, M_NOWAIT); if (dmat->segments == NULL) { CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); @@ -292,8 +317,8 @@ } bz = dmat->bounce_zone; - *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF, - M_NOWAIT | M_ZERO); + *mapp = (bus_dmamap_t)malloc_domain(sizeof(**mapp), M_DEVBUF, + dmat->common.domain, M_NOWAIT | M_ZERO); if (*mapp == NULL) { CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); @@ -355,7 +380,7 @@ } if (dmat->bounce_zone) dmat->bounce_zone->map_count--; - free(map, M_DEVBUF); + free_domain(map, M_DEVBUF); } dmat->map_count--; CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); @@ -386,9 +411,9 @@ *mapp = NULL; if (dmat->segments == NULL) { - dmat->segments = (bus_dma_segment_t *)malloc( + dmat->segments = (bus_dma_segment_t *)malloc_domain( sizeof(bus_dma_segment_t) * dmat->common.nsegments, - M_DEVBUF, mflags); + M_DEVBUF, dmat->common.domain, mflags); if (dmat->segments == NULL) { CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", __func__, dmat, dmat->common.flags, ENOMEM); @@ -427,18 +452,19 @@ (dmat->common.alignment <= dmat->common.maxsize) && dmat->common.lowaddr >= ptoa((vm_paddr_t)Maxmem) && attr == VM_MEMATTR_DEFAULT) { - *vaddr = malloc(dmat->common.maxsize, M_DEVBUF, mflags); + *vaddr = malloc_domain(dmat->common.maxsize, M_DEVBUF, + dmat->common.domain, mflags); } else if (dmat->common.nsegments >= howmany(dmat->common.maxsize, MIN(dmat->common.maxsegsz, PAGE_SIZE)) && dmat->common.alignment <= PAGE_SIZE && (dmat->common.boundary % PAGE_SIZE) == 0) { /* Page-based multi-segment allocations allowed */ - *vaddr = (void *)kmem_alloc_attr(kernel_arena, + *vaddr = (void *)kmem_alloc_attr_domain(dmat->common.domain, dmat->common.maxsize, mflags, 0ul, dmat->common.lowaddr, attr); dmat->bounce_flags |= BUS_DMA_KMEM_ALLOC; } else { - *vaddr = (void *)kmem_alloc_contig(kernel_arena, + *vaddr = (void *)kmem_alloc_contig_domain(dmat->common.domain, dmat->common.maxsize, mflags, 0ul, dmat->common.lowaddr, dmat->common.alignment != 0 ? dmat->common.alignment : 1ul, dmat->common.boundary, attr); @@ -471,7 +497,7 @@ if (map != NULL) panic("bus_dmamem_free: Invalid map freed\n"); if ((dmat->bounce_flags & BUS_DMA_KMEM_ALLOC) == 0) - free(vaddr, M_DEVBUF); + free_domain(vaddr, M_DEVBUF); else kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->common.maxsize); @@ -1041,7 +1067,8 @@ /* Check to see if we already have a suitable zone */ STAILQ_FOREACH(bz, &bounce_zone_list, links) { if ((dmat->common.alignment <= bz->alignment) && - (dmat->common.lowaddr >= bz->lowaddr)) { + (dmat->common.lowaddr >= bz->lowaddr) && + (dmat->common.domain == bz->domain)) { dmat->bounce_zone = bz; return (0); } @@ -1058,6 +1085,7 @@ bz->lowaddr = dmat->common.lowaddr; bz->alignment = MAX(dmat->common.alignment, PAGE_SIZE); bz->map_count = 0; + bz->domain = dmat->common.domain; snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); busdma_zonecount++; snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); @@ -1103,6 +1131,10 @@ SYSCTL_ADD_UAUTO(busdma_sysctl_tree(bz), SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, "alignment", CTLFLAG_RD, &bz->alignment, ""); + SYSCTL_ADD_INT(busdma_sysctl_tree(bz), + SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, + "domain", CTLFLAG_RD, &bz->domain, 0, + "memory domain"); return (0); } @@ -1118,18 +1150,16 @@ while (numpages > 0) { struct bounce_page *bpage; - bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, - M_NOWAIT | M_ZERO); + bpage = (struct bounce_page *)malloc_domain(sizeof(*bpage), + M_DEVBUF, dmat->common.domain, M_NOWAIT | M_ZERO); if (bpage == NULL) break; - bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, - M_NOWAIT, 0ul, - bz->lowaddr, - PAGE_SIZE, - 0); + bpage->vaddr = (vm_offset_t)contigmalloc_domain(PAGE_SIZE, + M_DEVBUF, dmat->common.domain, M_NOWAIT, 0ul, + bz->lowaddr, PAGE_SIZE, 0); if (bpage->vaddr == 0) { - free(bpage, M_DEVBUF); + free_domain(bpage, M_DEVBUF); break; } bpage->busaddr = pmap_kextract(bpage->vaddr); @@ -1271,6 +1301,7 @@ struct bus_dma_impl bus_dma_bounce_impl = { .tag_create = bounce_bus_dma_tag_create, .tag_destroy = bounce_bus_dma_tag_destroy, + .tag_set_domain = bounce_bus_dma_tag_set_domain, .map_create = bounce_bus_dmamap_create, .map_destroy = bounce_bus_dmamap_destroy, .mem_alloc = bounce_bus_dmamem_alloc, @@ -1281,5 +1312,5 @@ .map_waitok = bounce_bus_dmamap_waitok, .map_complete = bounce_bus_dmamap_complete, .map_unload = bounce_bus_dmamap_unload, - .map_sync = bounce_bus_dmamap_sync + .map_sync = bounce_bus_dmamap_sync, }; Index: x86/x86/busdma_machdep.c =================================================================== --- x86/x86/busdma_machdep.c +++ x86/x86/busdma_machdep.c @@ -43,8 +43,12 @@ #include #include #include +#include #include #include +#include +#include +#include #include #include #include @@ -180,10 +184,27 @@ common->filterarg = parent->filterarg; common->parent = parent->parent; } + common->domain = parent->domain; atomic_add_int(&parent->ref_count, 1); } + common->domain = vm_phys_domain_match(common->domain, 0ul, + common->lowaddr); *dmat = common; return (0); +} + +int +bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain) +{ + struct bus_dma_tag_common *tc; + + tc = (struct bus_dma_tag_common *)dmat; + domain = vm_phys_domain_match(domain, 0ul, tc->lowaddr); + /* Only call the callback if it changes. */ + if (domain == tc->domain) + return (0); + tc->domain = domain; + return (tc->impl->tag_set_domain(dmat)); } /*