Index: sys/vm/vm_phys.c =================================================================== --- sys/vm/vm_phys.c +++ sys/vm/vm_phys.c @@ -100,11 +100,12 @@ RB_INITIALIZER(&vm_phys_fictitious_tree); struct vm_phys_fictitious_seg { - RB_ENTRY(vm_phys_fictitious_seg) node; /* Memory region data */ vm_paddr_t start; vm_paddr_t end; vm_page_t first_page; + + RB_ENTRY(vm_phys_fictitious_seg) node; }; RB_GENERATE_STATIC(fict_tree, vm_phys_fictitious_seg, node, @@ -928,6 +929,12 @@ return (NULL); } +static inline vm_page_t +vm_seg_addr_to_page(struct vm_phys_seg *seg, vm_paddr_t pa) +{ + return (&seg->first_page[atop(pa - seg->start)]); +} + /* * Find the vm_page corresponding to the given physical address. */ @@ -937,7 +944,7 @@ struct vm_phys_seg *seg; if ((seg = vm_phys_paddr_to_seg(pa)) != NULL) - return (&seg->first_page[atop(pa - seg->start)]); + return (vm_seg_addr_to_page(seg, pa)); return (NULL); } @@ -957,7 +964,7 @@ if (seg == NULL) return (NULL); - m = &seg->first_page[atop(pa - seg->start)]; + m = vm_seg_addr_to_page((struct vm_phys_seg *)seg, pa); KASSERT((m->flags & PG_FICTITIOUS) != 0, ("%p not fictitious", m)); return (m); @@ -1153,7 +1160,7 @@ pa ^= ((vm_paddr_t)1 << (PAGE_SHIFT + order)); if (pa < seg->start || pa >= seg->end) break; - m_buddy = &seg->first_page[atop(pa - seg->start)]; + m_buddy = vm_seg_addr_to_page(seg, pa); if (m_buddy->order != order) break; fl = (*seg->free_queues)[m_buddy->pool]; @@ -1162,7 +1169,7 @@ vm_phys_set_pool(m->pool, m_buddy, order); order++; pa &= ~(((vm_paddr_t)1 << (PAGE_SHIFT + order)) - 1); - m = &seg->first_page[atop(pa - seg->start)]; + m = vm_seg_addr_to_page(seg, pa); } while (order < VM_NFREEORDER - 1); } fl = (*seg->free_queues)[m->pool]; @@ -1288,8 +1295,8 @@ pa_end = MIN(high, seg->end); if (pa_end - pa_start < ptoa(npages)) continue; - bounds[0] = &seg->first_page[atop(pa_start - seg->start)]; - bounds[1] = &seg->first_page[atop(pa_end - seg->start)]; + bounds[0] = vm_seg_addr_to_page(seg, pa_start); + bounds[1] = vm_seg_addr_to_page(seg, pa_end); return (seg - vm_phys_segs); } return (-1); @@ -1323,7 +1330,7 @@ order++; pa = m->phys_addr & (~(vm_paddr_t)0 << (PAGE_SHIFT + order)); if (pa >= seg->start) - m_set = &seg->first_page[atop(pa - seg->start)]; + m_set = vm_seg_addr_to_page(seg, pa); else return (false); } @@ -1348,10 +1355,10 @@ order--; pa_half = m_set->phys_addr ^ (1 << (PAGE_SHIFT + order)); if (m->phys_addr < pa_half) - m_tmp = &seg->first_page[atop(pa_half - seg->start)]; + m_tmp = vm_seg_addr_to_page(seg, pa_half); else { m_tmp = m_set; - m_set = &seg->first_page[atop(pa_half - seg->start)]; + m_set = vm_seg_addr_to_page(seg, pa_half); } vm_freelist_add(fl, m_tmp, order, 0); } @@ -1363,105 +1370,58 @@ * Find a run of contiguous physical pages from the specified page list. */ static vm_page_t -vm_phys_find_freelist_contig(struct vm_freelist *fl, int oind, u_long npages, +vm_phys_find_freelist_contig(struct vm_freelist *fl, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary) { struct vm_phys_seg *seg; - vm_paddr_t frag, lbound, pa, page_size, pa_end, pa_pre, size; - vm_page_t m, m_listed, m_ret; - int order; + vm_page_t m, m_end, m_ret; + vm_paddr_t max_size, size; + int max_order; - KASSERT(npages > 0, ("npages is 0")); - KASSERT(powerof2(alignment), ("alignment is not a power of 2")); - KASSERT(powerof2(boundary), ("boundary is not a power of 2")); - /* Search for a run satisfying the specified conditions. */ - page_size = PAGE_SIZE; + max_order = VM_NFREEORDER - 1; size = npages << PAGE_SHIFT; - frag = (npages & ~(~0UL << oind)) << PAGE_SHIFT; - TAILQ_FOREACH(m_listed, &fl[oind].pl, listq) { + max_size = (vm_paddr_t)1 << (PAGE_SHIFT + max_order); + TAILQ_FOREACH(m, &fl[max_order].pl, listq) { /* - * Determine if the address range starting at pa is - * too low. + * Skip m unless it is first in a sequence of free max page + * blocks >= low in its segment. */ - pa = VM_PAGE_TO_PHYS(m_listed); - if (pa < low) + seg = &vm_phys_segs[m->segind]; + if (VM_PAGE_TO_PHYS(m) < MAX(low, seg->start)) continue; - - /* - * If this is not the first free oind-block in this range, bail - * out. We have seen the first free block already, or will see - * it before failing to find an appropriate range. - */ - seg = &vm_phys_segs[m_listed->segind]; - lbound = low > seg->start ? low : seg->start; - pa_pre = pa - (page_size << oind); - m = &seg->first_page[atop(pa_pre - seg->start)]; - if (pa != 0 && pa_pre >= lbound && m->order == oind) + if (VM_PAGE_TO_PHYS(m) >= max_size && + VM_PAGE_TO_PHYS(m) - max_size >= MAX(low, seg->start) && + max_order == m[-1 << max_order].order) continue; - if (!vm_addr_align_ok(pa, alignment)) - /* Advance to satisfy alignment condition. */ - pa = roundup2(pa, alignment); - else if (frag != 0 && lbound + frag <= pa) { - /* - * Back up to the first aligned free block in this - * range, without moving below lbound. - */ - pa_end = pa; - for (order = oind - 1; order >= 0; order--) { - pa_pre = pa_end - (page_size << order); - if (!vm_addr_align_ok(pa_pre, alignment)) - break; - m = &seg->first_page[atop(pa_pre - seg->start)]; - if (pa_pre >= lbound && m->order == order) - pa_end = pa_pre; - } - /* - * If the extra small blocks are enough to complete the - * fragment, use them. Otherwise, look to allocate the - * fragment at the other end. - */ - if (pa_end + frag <= pa) - pa = pa_end; - } - - /* Advance as necessary to satisfy boundary conditions. */ - if (!vm_addr_bound_ok(pa, size, boundary)) - pa = roundup2(pa + 1, boundary); - pa_end = pa + size; - /* - * Determine if the address range is valid (without overflow in - * pa_end calculation), and fits within the segment. + * Advance m_ret from m to the first of the sequence, if any, + * that satisfies alignment conditions and might leave enough + * space. */ - if (pa_end < pa || seg->end < pa_end) + m_ret = m; + while (!vm_addr_ok(VM_PAGE_TO_PHYS(m_ret), + size, alignment, boundary) && + VM_PAGE_TO_PHYS(m_ret + npages) <= MIN(high, seg->end) && + max_order == m_ret[1 << max_order].order) + m_ret += 1 << max_order; + + m_end = m_ret + npages; + if (!vm_addr_ok(VM_PAGE_TO_PHYS(m_ret), + size, alignment, boundary) || + VM_PAGE_TO_PHYS(m_end) > MIN(high, seg->end)) continue; - m_ret = &seg->first_page[atop(pa - seg->start)]; - /* - * Determine whether there are enough free oind-blocks here to - * satisfy the allocation request. + * Return if the blocks to allocate starting at m_ret are all + * free. */ - pa = VM_PAGE_TO_PHYS(m_listed); + m = m_ret; do { - pa += page_size << oind; - if (pa >= pa_end) + m += 1 << max_order; + if (m >= m_end) return (m_ret); - m = &seg->first_page[atop(pa - seg->start)]; - } while (oind == m->order); - - /* - * Determine if an additional series of free blocks of - * diminishing size can help to satisfy the allocation request. - */ - while (m->order < oind && - pa + 2 * (page_size << m->order) > pa_end) { - pa += page_size << m->order; - if (pa >= pa_end) - return (m_ret); - m = &seg->first_page[atop(pa - seg->start)]; - } + } while (max_order == m->order); } return (NULL); } @@ -1509,10 +1469,9 @@ if (order < VM_NFREEORDER) return (NULL); /* Search for a long-enough sequence of small blocks. */ - oind = VM_NFREEORDER - 1; for (pind = 0; pind < VM_NFREEPOOL; pind++) { fl = (*queues)[pind]; - m_ret = vm_phys_find_freelist_contig(fl, oind, npages, + m_ret = vm_phys_find_freelist_contig(fl, npages, low, high, alignment, boundary); if (m_ret != NULL) return (m_ret); @@ -1521,14 +1480,14 @@ } /* - * Allocate a contiguous set of physical pages of the given size - * "npages" from the free lists. All of the physical pages must be at - * or above the given physical address "low" and below the given - * physical address "high". The given value "alignment" determines the - * alignment of the first physical page in the set. If the given value - * "boundary" is non-zero, then the set of physical pages cannot cross - * any physical address boundary that is a multiple of that value. Both - * "alignment" and "boundary" must be a power of two. + * Allocate a contiguous set of physical pages of the given size "npages" from + * the free lists of the given domain. All of the physical pages must be at or + * above the given physical address "low" and below the given physical address + * "high". The given value "alignment" determines the alignment of the first + * physical page in the set. If the given value "boundary" is non-zero, then + * the set of physical pages cannot cross any physical address boundary that is + * a multiple of that value. Both "alignment" and "boundary" must be a power of + * two. */ vm_page_t vm_phys_alloc_contig(int domain, u_long npages, vm_paddr_t low, vm_paddr_t high, @@ -1574,6 +1533,9 @@ if (seg->free_queues == queues) continue; queues = seg->free_queues; + + /* All free list items are associated with 'domain'. */ + vm_domain_free_assert_locked(VM_DOMAIN(domain)); m_run = vm_phys_find_queues_contig(queues, npages, low, high, alignment, boundary); if (m_run != NULL) @@ -1593,6 +1555,19 @@ /* Return excess pages to the free lists. */ fl = (*queues)[VM_FREEPOOL_DEFAULT]; vm_phys_enq_range(&m_run[npages], m - &m_run[npages], fl, 0); + + /* Return page verified to satisfy conditions of request. */ + pa_start = VM_PAGE_TO_PHYS(m_run); + KASSERT(low <= pa_start, + ("memory allocated below minimum requested range")); + pa_end = VM_PAGE_TO_PHYS(m_run + npages); + KASSERT(pa_end <= high, + ("memory allocated above maximum requested range")); + seg = vm_phys_paddr_to_seg(pa_start); + KASSERT(seg != NULL && seg->domain == domain, + ("memory not allocated from specified domain")); + KASSERT(vm_addr_ok(pa_start, npages << PAGE_SHIFT, alignment, boundary), + ("memory alignment/boundary contraints not satisfied")); return (m_run); }