Index: sys/vm/vm_phys.c =================================================================== --- sys/vm/vm_phys.c +++ sys/vm/vm_phys.c @@ -100,11 +100,12 @@ RB_INITIALIZER(&vm_phys_fictitious_tree); struct vm_phys_fictitious_seg { - RB_ENTRY(vm_phys_fictitious_seg) node; /* Memory region data */ vm_paddr_t start; vm_paddr_t end; vm_page_t first_page; + + RB_ENTRY(vm_phys_fictitious_seg) node; }; RB_GENERATE_STATIC(fict_tree, vm_phys_fictitious_seg, node, @@ -928,6 +929,12 @@ return (NULL); } +static inline vm_page_t +vm_seg_addr_to_page(struct vm_phys_seg *seg, vm_paddr_t pa) +{ + return (&seg->first_page[atop(pa - seg->start)]); +} + /* * Find the vm_page corresponding to the given physical address. */ @@ -937,7 +944,7 @@ struct vm_phys_seg *seg; if ((seg = vm_phys_paddr_to_seg(pa)) != NULL) - return (&seg->first_page[atop(pa - seg->start)]); + return (vm_seg_addr_to_page(seg, pa)); return (NULL); } @@ -957,7 +964,7 @@ if (seg == NULL) return (NULL); - m = &seg->first_page[atop(pa - seg->start)]; + m = vm_seg_addr_to_page((struct vm_phys_seg *)seg, pa); KASSERT((m->flags & PG_FICTITIOUS) != 0, ("%p not fictitious", m)); return (m); @@ -1153,7 +1160,7 @@ pa ^= ((vm_paddr_t)1 << (PAGE_SHIFT + order)); if (pa < seg->start || pa >= seg->end) break; - m_buddy = &seg->first_page[atop(pa - seg->start)]; + m_buddy = vm_seg_addr_to_page(seg, pa); if (m_buddy->order != order) break; fl = (*seg->free_queues)[m_buddy->pool]; @@ -1162,7 +1169,7 @@ vm_phys_set_pool(m->pool, m_buddy, order); order++; pa &= ~(((vm_paddr_t)1 << (PAGE_SHIFT + order)) - 1); - m = &seg->first_page[atop(pa - seg->start)]; + m = vm_seg_addr_to_page(seg, pa); } while (order < VM_NFREEORDER - 1); } fl = (*seg->free_queues)[m->pool]; @@ -1288,8 +1295,8 @@ pa_end = MIN(high, seg->end); if (pa_end - pa_start < ptoa(npages)) continue; - bounds[0] = &seg->first_page[atop(pa_start - seg->start)]; - bounds[1] = &seg->first_page[atop(pa_end - seg->start)]; + bounds[0] = vm_seg_addr_to_page(seg, pa_start); + bounds[1] = vm_seg_addr_to_page(seg, pa_end); return (seg - vm_phys_segs); } return (-1); @@ -1323,7 +1330,7 @@ order++; pa = m->phys_addr & (~(vm_paddr_t)0 << (PAGE_SHIFT + order)); if (pa >= seg->start) - m_set = &seg->first_page[atop(pa - seg->start)]; + m_set = vm_seg_addr_to_page(seg, pa); else return (false); } @@ -1348,10 +1355,10 @@ order--; pa_half = m_set->phys_addr ^ (1 << (PAGE_SHIFT + order)); if (m->phys_addr < pa_half) - m_tmp = &seg->first_page[atop(pa_half - seg->start)]; + m_tmp = vm_seg_addr_to_page(seg, pa_half); else { m_tmp = m_set; - m_set = &seg->first_page[atop(pa_half - seg->start)]; + m_set = vm_seg_addr_to_page(seg, pa_half); } vm_freelist_add(fl, m_tmp, order, 0); } @@ -1363,105 +1370,55 @@ * Find a run of contiguous physical pages from the specified page list. */ static vm_page_t -vm_phys_find_freelist_contig(struct vm_freelist *fl, int oind, u_long npages, +vm_phys_find_freelist_contig(struct vm_freelist *fl, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary) { struct vm_phys_seg *seg; - vm_paddr_t frag, lbound, pa, page_size, pa_end, pa_pre, size; - vm_page_t m, m_listed, m_ret; - int order; + vm_page_t m_ret, m_val; + vm_paddr_t max_size, pa, pa_end, size; - KASSERT(npages > 0, ("npages is 0")); - KASSERT(powerof2(alignment), ("alignment is not a power of 2")); - KASSERT(powerof2(boundary), ("boundary is not a power of 2")); - /* Search for a run satisfying the specified conditions. */ - page_size = PAGE_SIZE; size = npages << PAGE_SHIFT; - frag = (npages & ~(~0UL << oind)) << PAGE_SHIFT; - TAILQ_FOREACH(m_listed, &fl[oind].pl, listq) { + max_size = (vm_paddr_t)1 << (PAGE_SHIFT + VM_NFREEORDER - 1); + TAILQ_FOREACH(m_ret, &fl[VM_NFREEORDER - 1].pl, listq) { /* - * Determine if the address range starting at pa is - * too low. + * Verify that pa is the first free max page block >= low in the + * segment and in the sequence of physically contiguous blocks + * that contains pa. */ - pa = VM_PAGE_TO_PHYS(m_listed); - if (pa < low) + pa = VM_PAGE_TO_PHYS(m_ret); + seg = &vm_phys_segs[m_ret->segind]; + if (pa < MAX(low, seg->start)) continue; - - /* - * If this is not the first free oind-block in this range, bail - * out. We have seen the first free block already, or will see - * it before failing to find an appropriate range. - */ - seg = &vm_phys_segs[m_listed->segind]; - lbound = low > seg->start ? low : seg->start; - pa_pre = pa - (page_size << oind); - m = &seg->first_page[atop(pa_pre - seg->start)]; - if (pa != 0 && pa_pre >= lbound && m->order == oind) + if (pa >= max_size && pa - max_size >= MAX(low, seg->start) && + VM_NFREEORDER - 1 == + vm_seg_addr_to_page(seg, pa - max_size)->order) continue; - if (!vm_addr_align_ok(pa, alignment)) - /* Advance to satisfy alignment condition. */ - pa = roundup2(pa, alignment); - else if (frag != 0 && lbound + frag <= pa) { - /* - * Back up to the first aligned free block in this - * range, without moving below lbound. - */ - pa_end = pa; - for (order = oind - 1; order >= 0; order--) { - pa_pre = pa_end - (page_size << order); - if (!vm_addr_align_ok(pa_pre, alignment)) - break; - m = &seg->first_page[atop(pa_pre - seg->start)]; - if (pa_pre >= lbound && m->order == order) - pa_end = pa_pre; - } - /* - * If the extra small blocks are enough to complete the - * fragment, use them. Otherwise, look to allocate the - * fragment at the other end. - */ - if (pa_end + frag <= pa) - pa = pa_end; - } - - /* Advance as necessary to satisfy boundary conditions. */ - if (!vm_addr_bound_ok(pa, size, boundary)) - pa = roundup2(pa + 1, boundary); - pa_end = pa + size; - /* - * Determine if the address range is valid (without overflow in - * pa_end calculation), and fits within the segment. + * Advance pa to the first of the sequence, if any, that + * satisfies alignment conditions and might leave enough space. */ - if (pa_end < pa || seg->end < pa_end) + while (!vm_addr_ok(pa, size, alignment, boundary) && + pa + size <= MIN(high, seg->end) && + VM_NFREEORDER - 1 == + vm_seg_addr_to_page(seg, pa + max_size)->order) + pa += max_size; + pa_end = pa + size; + if (!vm_addr_ok(pa, size, alignment, boundary) || + pa_end > MIN(high, seg->end) || pa_end < pa) continue; - - m_ret = &seg->first_page[atop(pa - seg->start)]; + m_val = vm_seg_addr_to_page(seg, pa); /* - * Determine whether there are enough free oind-blocks here to - * satisfy the allocation request. + * Verify that the blocks to allocate starting at pa are all + * free. */ - pa = VM_PAGE_TO_PHYS(m_listed); do { - pa += page_size << oind; + pa += max_size; if (pa >= pa_end) - return (m_ret); - m = &seg->first_page[atop(pa - seg->start)]; - } while (oind == m->order); - - /* - * Determine if an additional series of free blocks of - * diminishing size can help to satisfy the allocation request. - */ - while (m->order < oind && - pa + 2 * (page_size << m->order) > pa_end) { - pa += page_size << m->order; - if (pa >= pa_end) - return (m_ret); - m = &seg->first_page[atop(pa - seg->start)]; - } + return (m_val); + } while (VM_NFREEORDER - 1 == + vm_seg_addr_to_page(seg, pa)->order); } return (NULL); } @@ -1509,10 +1466,9 @@ if (order < VM_NFREEORDER) return (NULL); /* Search for a long-enough sequence of small blocks. */ - oind = VM_NFREEORDER - 1; for (pind = 0; pind < VM_NFREEPOOL; pind++) { fl = (*queues)[pind]; - m_ret = vm_phys_find_freelist_contig(fl, oind, npages, + m_ret = vm_phys_find_freelist_contig(fl, npages, low, high, alignment, boundary); if (m_ret != NULL) return (m_ret); @@ -1574,6 +1530,7 @@ if (seg->free_queues == queues) continue; queues = seg->free_queues; + vm_domain_free_assert_locked(VM_DOMAIN(seg->domain)); m_run = vm_phys_find_queues_contig(queues, npages, low, high, alignment, boundary); if (m_run != NULL)