Index: vm_page.c =================================================================== --- vm_page.c +++ vm_page.c @@ -833,7 +833,7 @@ vmd = VM_DOMAIN(seg->domain); vm_domain_free_lock(vmd); - vm_phys_free_contig(m, pagecount); + vm_phys_free_bounded_contig(m, pagecount); vm_domain_free_unlock(vmd); vm_domain_freecnt_inc(vmd, pagecount); vm_cnt.v_page_count += (u_int)pagecount; Index: vm_phys.h =================================================================== --- vm_phys.h +++ vm_phys.h @@ -88,6 +88,7 @@ vm_memattr_t memattr); void vm_phys_fictitious_unreg_range(vm_paddr_t start, vm_paddr_t end); vm_page_t vm_phys_fictitious_to_vm_page(vm_paddr_t pa); +void vm_phys_free_bounded_contig(vm_page_t m, u_long npages); void vm_phys_free_contig(vm_page_t m, u_long npages); void vm_phys_free_pages(vm_page_t m, int order); void vm_phys_init(void); Index: vm_phys.c =================================================================== --- vm_phys.c +++ vm_phys.c @@ -1095,14 +1095,38 @@ } /* - * Free a contiguous, arbitrarily sized set of physical pages. + * Return the largest possible order of a page starting at m. + */ +static int +max_order(vm_page_t m) +{ + /* + * Unsigned "min" is used here so that "order" is assigned + * "VM_NFREEORDER - 1" when "m"'s physical address is zero + * or the low-order bits of its physical address are zero + * because the size of a physical address exceeds the size of + * a long. + */ + return (min(ffsl(VM_PAGE_TO_PHYS(m) >> PAGE_SHIFT) - 1, + VM_NFREEORDER - 1)); +} + +/* + * Free a contiguous, arbitrarily sized set of physical pages, without + * merging across set boundaries. * + * Either the pages just across a boundary must be allocated, or the boundary + * must be superpage boundary, because there is no check here for + * buddy-system merges across a boundary. + * * The free page queues must be locked. */ void -vm_phys_free_contig(vm_page_t m, u_long npages) +vm_phys_free_bounded_contig(vm_page_t m, u_long npages) { - u_int n; + struct vm_freelist *fl; + struct vm_phys_seg *seg; + vm_page_t m_end; int order; /* @@ -1110,32 +1134,71 @@ * possible power-of-two-sized subsets. */ vm_domain_free_assert_locked(vm_pagequeue_domain(m)); - for (;; npages -= n) { - /* - * Unsigned "min" is used here so that "order" is assigned - * "VM_NFREEORDER - 1" when "m"'s physical address is zero - * or the low-order bits of its physical address are zero - * because the size of a physical address exceeds the size of - * a long. - */ - order = min(ffsl(VM_PAGE_TO_PHYS(m) >> PAGE_SHIFT) - 1, - VM_NFREEORDER - 1); - n = 1 << order; - if (npages < n) - break; - vm_phys_free_pages(m, order); - m += n; + seg = &vm_phys_segs[m->segind]; + fl = (*seg->free_queues)[m->pool]; + m_end = m + npages; + /* Free blocks of increasing size. */ + while ((order = max_order(m)) < VM_NFREEORDER - 1 && + m + (1 << order) <= m_end) { + KASSERT(seg == &vm_phys_segs[m->segind], + ("%s: page range [%p,%p) spans multiple segments", + __func__, m, m_end)); + vm_freelist_add(fl, m, order, 1); + m += (1 << order); } - /* The residual "npages" is less than "1 << (VM_NFREEORDER - 1)". */ - for (; npages > 0; npages -= n) { - order = flsl(npages) - 1; - n = 1 << order; - vm_phys_free_pages(m, order); - m += n; + /* Free blocks of maximum size. */ + while (m + (1 << order) <= m_end) { + KASSERT(seg == &vm_phys_segs[m->segind], + ("%s: page range [%p,%p) spans multiple segments", + __func__, m, m_end)); + vm_freelist_add(fl, m, order, 1); + m += (1 << order); } + /* Free blocks of diminishing size. */ + while (m < m_end) { + KASSERT(seg == &vm_phys_segs[m->segind], + ("%s: page range [%p,%p) spans multiple segments", + __func__, m, m_end)); + order = flsl(m_end - m) - 1; + vm_freelist_add(fl, m, order, 1); + m += (1 << order); + } } /* + * Free a contiguous, arbitrarily sized set of physical pages. + * + * The free page queues must be locked. + */ +void +vm_phys_free_contig(vm_page_t m, u_long npages) +{ + int order_start, order_end; + vm_page_t m_start, m_end; + + vm_domain_free_assert_locked(vm_pagequeue_domain(m)); + + m_start = m; + order_start = max_order(m_start); + if (order_start < VM_NFREEORDER - 1) + m_start += 1 << order_start; + m_end = m + npages; + order_end = max_order(m_end); + if (order_end < VM_NFREEORDER - 1) + m_end -= 1 << order_end; + /* + * Avoid unnecessary coalescing by freeing the pages at the start and + * end of the range last. + */ + if (m_start < m_end) + vm_phys_free_bounded_contig(m_start, m_end - m_start); + if (order_start < VM_NFREEORDER - 1) + vm_phys_free_pages(m, order_start); + if (order_end < VM_NFREEORDER - 1) + vm_phys_free_pages(m_end, order_end); +} + +/* * Scan physical memory between the specified addresses "low" and "high" for a * run of contiguous physical pages that satisfy the specified conditions, and * return the lowest page in the run. The specified "alignment" determines Index: vm_reserv.c =================================================================== --- vm_reserv.c +++ vm_reserv.c @@ -1066,7 +1066,8 @@ else { hi = NBPOPMAP * i + bitpos; vm_domain_free_lock(VM_DOMAIN(rv->domain)); - vm_phys_free_contig(&rv->pages[lo], hi - lo); + vm_phys_free_bounded_contig(&rv->pages[lo], + hi - lo); vm_domain_free_unlock(VM_DOMAIN(rv->domain)); lo = hi; }