Index: sys/vm/uma_core.c =================================================================== --- sys/vm/uma_core.c +++ sys/vm/uma_core.c @@ -1954,17 +1954,15 @@ pcpu_page_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, int wait) { - struct pglist alloctail; - vm_offset_t addr, zkva; + vm_offset_t addr; int cpu, flags; - vm_page_t p, p_next; + vm_page_t p, pa[mp_maxid + 1]; #ifdef NUMA struct pcpu *pc; #endif MPASS(bytes == (mp_maxid + 1) * PAGE_SIZE); - TAILQ_INIT(&alloctail); flags = VM_ALLOC_SYSTEM | VM_ALLOC_WIRED | malloc2vm_flags(wait); *pflag = UMA_SLAB_KERNEL; for (cpu = 0; cpu <= mp_maxid; cpu++) { @@ -1986,20 +1984,16 @@ } if (__predict_false(p == NULL)) goto fail; - TAILQ_INSERT_TAIL(&alloctail, p, listq); + pa[cpu] = p; } if ((addr = kva_alloc(bytes)) == 0) goto fail; - zkva = addr; - TAILQ_FOREACH(p, &alloctail, listq) { - pmap_qenter(zkva, &p, 1); - zkva += PAGE_SIZE; - } + pmap_qenter(addr, pa, mp_maxid + 1); return ((void*)addr); fail: - TAILQ_FOREACH_SAFE(p, &alloctail, listq, p_next) { - vm_page_unwire_noq(p); - vm_page_free(p); + while (cpu-- > 0) { + vm_page_unwire_noq(pa[cpu]); + vm_page_free(pa[cpu]); } return (NULL); } @@ -2019,51 +2013,38 @@ noobj_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags, int wait) { - TAILQ_HEAD(, vm_page) alloctail; - u_long npages; - vm_offset_t retkva, zkva; - vm_page_t p, p_next; + u_long npages = howmany(bytes, PAGE_SIZE); + vm_offset_t zkva; + vm_page_t p, pa[npages]; uma_keg_t keg; int req; - TAILQ_INIT(&alloctail); keg = zone->uz_keg; req = VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED; if ((wait & M_WAITOK) != 0) req |= VM_ALLOC_WAITOK; - npages = howmany(bytes, PAGE_SIZE); - while (npages > 0) { + for (u_long i = 0; i < npages; i++) { p = vm_page_alloc_noobj_domain(domain, req); - if (p != NULL) { + if (p == NULL) { /* - * Since the page does not belong to an object, its - * listq is unused. + * Page allocation failed, free intermediate pages and + * exit. */ - TAILQ_INSERT_TAIL(&alloctail, p, listq); - npages--; - continue; - } - /* - * Page allocation failed, free intermediate pages and - * exit. - */ - TAILQ_FOREACH_SAFE(p, &alloctail, listq, p_next) { - vm_page_unwire_noq(p); - vm_page_free(p); + while (i-- > 0) { + vm_page_unwire_noq(pa[i]); + vm_page_free(pa[i]); + } + return (NULL); } - return (NULL); + pa[i] = p; } *flags = UMA_SLAB_PRIV; zkva = keg->uk_kva + atomic_fetchadd_long(&keg->uk_offset, round_page(bytes)); - retkva = zkva; - TAILQ_FOREACH(p, &alloctail, listq) { - pmap_qenter(zkva, &p, 1); - zkva += PAGE_SIZE; - } + pmap_qenter(zkva, pa, npages); - return ((void *)retkva); + return ((void *)zkva); } /*