Changeset View
Standalone View
sys/vm/vm_phys.c
| Show First 20 Lines • Show All 951 Lines • ▼ Show 20 Lines | vm_phys_fictitious_to_vm_page(vm_paddr_t pa) | |||||||||||
| tmp.end = 0; | tmp.end = 0; | |||||||||||
| rw_rlock(&vm_phys_fictitious_reg_lock); | rw_rlock(&vm_phys_fictitious_reg_lock); | |||||||||||
| seg = RB_FIND(fict_tree, &vm_phys_fictitious_tree, &tmp); | seg = RB_FIND(fict_tree, &vm_phys_fictitious_tree, &tmp); | |||||||||||
| rw_runlock(&vm_phys_fictitious_reg_lock); | rw_runlock(&vm_phys_fictitious_reg_lock); | |||||||||||
| if (seg == NULL) | if (seg == NULL) | |||||||||||
| return (NULL); | return (NULL); | |||||||||||
| m = &seg->first_page[atop(pa - seg->start)]; | m = &seg->first_page[atop(pa - seg->start)]; | |||||||||||
alc: This is asking for trouble from someone who doesn't carefully read all of the code. | ||||||||||||
| KASSERT((m->flags & PG_FICTITIOUS) != 0, ("%p not fictitious", m)); | KASSERT((m->flags & PG_FICTITIOUS) != 0, ("%p not fictitious", m)); | |||||||||||
| return (m); | return (m); | |||||||||||
| } | } | |||||||||||
| static inline void | static inline void | |||||||||||
| vm_phys_fictitious_init_range(vm_page_t range, vm_paddr_t start, | vm_phys_fictitious_init_range(vm_page_t range, vm_paddr_t start, | |||||||||||
| long page_count, vm_memattr_t memattr) | long page_count, vm_memattr_t memattr) | |||||||||||
| ▲ Show 20 Lines • Show All 386 Lines • ▼ Show 20 Lines | while (order > 0) { | |||||||||||
| } | } | |||||||||||
| vm_freelist_add(fl, m_tmp, order, 0); | vm_freelist_add(fl, m_tmp, order, 0); | |||||||||||
| } | } | |||||||||||
| KASSERT(m_set == m, ("vm_phys_unfree_page: fatal inconsistency")); | KASSERT(m_set == m, ("vm_phys_unfree_page: fatal inconsistency")); | |||||||||||
| return (true); | return (true); | |||||||||||
| } | } | |||||||||||
| /* | /* | |||||||||||
| * Find a run of contiguous physical pages from the specified page list. | * Find a run of contiguous physical pages, meeting alignment requirements, from | |||||||||||
| * a list of max-sized page blocks, where we need at least two consecutive | ||||||||||||
| * blocks to satisfy the (large) page request. | ||||||||||||
Not Done Inline Actions
markj: | ||||||||||||
Not Done Inline ActionsThis comment is not currently very descriptive of what this function does. alc: This comment is not currently very descriptive of what this function does. | ||||||||||||
| */ | */ | |||||||||||
| static vm_page_t | static vm_page_t | |||||||||||
| vm_phys_find_freelist_contig(struct vm_freelist *fl, int oind, u_long npages, | vm_phys_find_freelist_contig(struct vm_freelist *fl, u_long npages, | |||||||||||
| vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary) | vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary) | |||||||||||
| { | { | |||||||||||
| struct vm_phys_seg *seg; | struct vm_phys_seg *seg; | |||||||||||
| vm_paddr_t frag, lbound, pa, page_size, pa_end, pa_pre, size; | vm_page_t m, m_iter, m_ret; | |||||||||||
| vm_page_t m, m_listed, m_ret; | vm_paddr_t max_size, size; | |||||||||||
| int order; | int max_order; | |||||||||||
| KASSERT(npages > 0, ("npages is 0")); | max_order = VM_NFREEORDER - 1; | |||||||||||
| KASSERT(powerof2(alignment), ("alignment is not a power of 2")); | ||||||||||||
| KASSERT(powerof2(boundary), ("boundary is not a power of 2")); | ||||||||||||
| /* Search for a run satisfying the specified conditions. */ | ||||||||||||
| page_size = PAGE_SIZE; | ||||||||||||
| size = npages << PAGE_SHIFT; | size = npages << PAGE_SHIFT; | |||||||||||
| frag = (npages & ~(~0UL << oind)) << PAGE_SHIFT; | max_size = (vm_paddr_t)1 << (PAGE_SHIFT + max_order); | |||||||||||
Not Done Inline Actions
This describes what the code does, but not why. Specifically, why do we care that the preceding block is free? alc: This describes what the code does, but not why. Specifically, why do we care that the… | ||||||||||||
Not Done Inline ActionsI would KASSERT(size >= max_size, ... alc: I would KASSERT(size >= max_size, ... | ||||||||||||
Not Done Inline ActionsActually, strictly greater than. alc: Actually, strictly greater than. | ||||||||||||
| TAILQ_FOREACH(m_listed, &fl[oind].pl, listq) { | KASSERT(size > max_size, ("size is too small")); | |||||||||||
Not Done Inline Actions
or m_ret->phys_addr. markj: or `m_ret->phys_addr`. | ||||||||||||
| /* | /* | |||||||||||
| * Determine if the address range starting at pa is | * In order to avoid examining any free max-sized page block more than | |||||||||||
| * too low. | * twice, identify the ones that are first in a physically-continous | |||||||||||
alcUnsubmitted Not Done Inline Actions
alc: | ||||||||||||
| * sequence of such blocks, and only for those walk the sequence to | ||||||||||||
| * check if there are enough free blocks starting at a properly aligned | ||||||||||||
| * block. Thus, no block is checked for free-ness more than twice. | ||||||||||||
| */ | */ | |||||||||||
| pa = VM_PAGE_TO_PHYS(m_listed); | TAILQ_FOREACH(m, &fl[max_order].pl, listq) { | |||||||||||
| if (pa < low) | ||||||||||||
| continue; | ||||||||||||
| /* | /* | |||||||||||
| * If this is not the first free oind-block in this range, bail | * Skip m unless it is first in a sequence of free max page | |||||||||||
| * out. We have seen the first free block already, or will see | * blocks >= low in its segment. | |||||||||||
| * it before failing to find an appropriate range. | ||||||||||||
| */ | */ | |||||||||||
| seg = &vm_phys_segs[m_listed->segind]; | seg = &vm_phys_segs[m->segind]; | |||||||||||
| lbound = low > seg->start ? low : seg->start; | if (VM_PAGE_TO_PHYS(m) < MAX(low, seg->start)) | |||||||||||
| pa_pre = pa - (page_size << oind); | ||||||||||||
| m = &seg->first_page[atop(pa_pre - seg->start)]; | ||||||||||||
| if (pa != 0 && pa_pre >= lbound && m->order == oind) | ||||||||||||
| continue; | continue; | |||||||||||
| if (VM_PAGE_TO_PHYS(m) >= max_size && | ||||||||||||
| VM_PAGE_TO_PHYS(m) - max_size >= MAX(low, seg->start) && | ||||||||||||
| max_order == m[-1 << max_order].order) | ||||||||||||
| continue; | ||||||||||||
| if (!vm_addr_align_ok(pa, alignment)) | ||||||||||||
| /* Advance to satisfy alignment condition. */ | ||||||||||||
| pa = roundup2(pa, alignment); | ||||||||||||
| else if (frag != 0 && lbound + frag <= pa) { | ||||||||||||
| /* | /* | |||||||||||
| * Back up to the first aligned free block in this | * Advance m_ret from m to the first of the sequence, if any, | |||||||||||
| * range, without moving below lbound. | * that satisfies alignment conditions and might leave enough | |||||||||||
| */ | * space. | |||||||||||
Not Done Inline Actions
markj: | ||||||||||||
| pa_end = pa; | */ | |||||||||||
| for (order = oind - 1; order >= 0; order--) { | m_ret = m; | |||||||||||
| pa_pre = pa_end - (page_size << order); | while (!vm_addr_ok(VM_PAGE_TO_PHYS(m_ret), | |||||||||||
| if (!vm_addr_align_ok(pa_pre, alignment)) | size, alignment, boundary) && | |||||||||||
Not Done Inline ActionsThis assumes that there is an initialized page structure at m_ret + npages. That is not a valid assumption. You still need "pa" to perform the bounds checks. alc: This assumes that there is an initialized page structure at m_ret + npages. That is not a… | ||||||||||||
| break; | VM_PAGE_TO_PHYS(m_ret) + size <= MIN(high, seg->end) && | |||||||||||
Not Done Inline Actions
I don't see how 'size' is correct here. alc: I don't see how 'size' is correct here. | ||||||||||||
Not Done Inline ActionsYou're relying on size being greater than max_size. Okay. alc: You're relying on size being greater than max_size. Okay. | ||||||||||||
| m = &seg->first_page[atop(pa_pre - seg->start)]; | max_order == m_ret[1 << max_order].order) | |||||||||||
| if (pa_pre >= lbound && m->order == order) | m_ret += 1 << max_order; | |||||||||||
Not Done Inline ActionsI don't see the point of introducing a new variable, m_ret, alc: I don't see the point of introducing a new variable, m_ret, | ||||||||||||
Done Inline ActionsSuppose that m_ret is advanced in line 1400, and then that the FOREACH loop iteration ends with 'continue', rather than 'return'. Then the TAILQ_FOREACH advances m to the next item in the list. If there is no m_ret, and m was instead advanced in line 1400, where will the TAILQ_FOREACH take us next? Maybe it'll skip way ahead in the list. Maybe it'll skip back, so that we have an infinite loop. dougm: Suppose that m_ret is advanced in line 1400, and then that the FOREACH loop iteration ends with… | ||||||||||||
| pa_end = pa_pre; | ||||||||||||
| } | ||||||||||||
| /* | ||||||||||||
| * If the extra small blocks are enough to complete the | ||||||||||||
| * fragment, use them. Otherwise, look to allocate the | ||||||||||||
| * fragment at the other end. | ||||||||||||
| */ | ||||||||||||
| if (pa_end + frag <= pa) | ||||||||||||
| pa = pa_end; | ||||||||||||
| } | ||||||||||||
| /* Advance as necessary to satisfy boundary conditions. */ | ||||||||||||
| if (!vm_addr_bound_ok(pa, size, boundary)) | ||||||||||||
| pa = roundup2(pa + 1, boundary); | ||||||||||||
| pa_end = pa + size; | ||||||||||||
| /* | /* | |||||||||||
Not Done Inline ActionsThis really ought to have a comment saying that it ensures that the loop that follows is only looking at legitimate vm_page structures. alc: This really ought to have a comment saying that it ensures that the loop that follows is only… | ||||||||||||
| * Determine if the address range is valid (without overflow in | * Skip m unless some block m_ret in the sequence is properly aligned, | |||||||||||
| * pa_end calculation), and fits within the segment. | * and begins a sequence of enough initialized pages less than | |||||||||||
Not Done Inline Actions
alc: | ||||||||||||
| * high, and in the same segment. | ||||||||||||
Not Done Inline Actions
There's no reason to expect this condition to be true in general, so "verify" sounds a bit misleading. markj: There's no reason to expect this condition to be true in general, so "verify" sounds a bit… | ||||||||||||
| */ | */ | |||||||||||
| if (pa_end < pa || seg->end < pa_end) | if (!VM_PAGE_TO_PHYS(m_ret) + size <= MIN(high, seg->end)) | |||||||||||
| continue; | continue; | |||||||||||
Not Done Inline ActionsIs this check inverted? Certainly the ! looks wrong. markj: Is this check inverted? Certainly the `!` looks wrong. | ||||||||||||
| m_ret = &seg->first_page[atop(pa - seg->start)]; | ||||||||||||
| /* | /* | |||||||||||
| * Determine whether there are enough free oind-blocks here to | * Skip m unless the blocks to allocate starting at m_ret are | |||||||||||
| * satisfy the allocation request. | * all free. | |||||||||||
| */ | */ | |||||||||||
| pa = VM_PAGE_TO_PHYS(m_listed); | for (m_iter = m_ret; | |||||||||||
| do { | m_iter < m_ret + npages && max_order == m_iter->order; | |||||||||||
| pa += page_size << oind; | m_iter += 1 << max_order) { | |||||||||||
| if (pa >= pa_end) | } | |||||||||||
| return (m_ret); | if (m_iter < m_ret + npages) | |||||||||||
| m = &seg->first_page[atop(pa - seg->start)]; | continue; | |||||||||||
Not Done Inline ActionsRather than advancing pa for each blocking, translating it to a vm_page, and then checking its order, would it be marginally more efficient to simply advance a vm_page pointer by adding 1 << (VM_NFREEORDER - 1) to it in each iteration? markj: Rather than advancing `pa` for each blocking, translating it to a vm_page, and then checking… | ||||||||||||
Not Done Inline ActionsThe semi-colon at the end of the line is too easy to overlook. alc: The semi-colon at the end of the line is too easy to overlook. | ||||||||||||
| } while (oind == m->order); | ||||||||||||
| /* | ||||||||||||
| * Determine if an additional series of free blocks of | ||||||||||||
| * diminishing size can help to satisfy the allocation request. | ||||||||||||
| */ | ||||||||||||
| while (m->order < oind && | ||||||||||||
| pa + 2 * (page_size << m->order) > pa_end) { | ||||||||||||
| pa += page_size << m->order; | ||||||||||||
| if (pa >= pa_end) | ||||||||||||
| return (m_ret); | return (m_ret); | |||||||||||
Not Done Inline ActionsThe loop variable is called m_ret, but it is never returned. markj: The loop variable is called `m_ret`, but it is never returned. | ||||||||||||
| m = &seg->first_page[atop(pa - seg->start)]; | ||||||||||||
| } | } | |||||||||||
| } | ||||||||||||
| return (NULL); | return (NULL); | |||||||||||
| } | } | |||||||||||
| /* | /* | |||||||||||
| * Find a run of contiguous physical pages from the specified free list | * Find a run of contiguous physical pages from the specified free list | |||||||||||
| * table. | * table. | |||||||||||
| */ | */ | |||||||||||
| static vm_page_t | static vm_page_t | |||||||||||
| Show All 28 Lines | for (pind = 0; pind < VM_NFREEPOOL; pind++) { | |||||||||||
| pa_end = pa + size; | pa_end = pa + size; | |||||||||||
| if (low <= pa && pa_end <= high && | if (low <= pa && pa_end <= high && | |||||||||||
| vm_addr_ok(pa, size, alignment, boundary)) | vm_addr_ok(pa, size, alignment, boundary)) | |||||||||||
| return (m_ret); | return (m_ret); | |||||||||||
| } | } | |||||||||||
| } | } | |||||||||||
| } | } | |||||||||||
| if (order < VM_NFREEORDER) | if (order < VM_NFREEORDER) | |||||||||||
| return (NULL); | return (NULL); | |||||||||||
| /* Search for a long-enough sequence of small blocks. */ | /* Search for a long-enough sequence of max-order blocks. */ | |||||||||||
Not Done Inline Actions
alc: | ||||||||||||
| oind = VM_NFREEORDER - 1; | ||||||||||||
| for (pind = 0; pind < VM_NFREEPOOL; pind++) { | for (pind = 0; pind < VM_NFREEPOOL; pind++) { | |||||||||||
| fl = (*queues)[pind]; | fl = (*queues)[pind]; | |||||||||||
| m_ret = vm_phys_find_freelist_contig(fl, oind, npages, | m_ret = vm_phys_find_freelist_contig(fl, npages, | |||||||||||
| low, high, alignment, boundary); | low, high, alignment, boundary); | |||||||||||
| if (m_ret != NULL) | if (m_ret != NULL) | |||||||||||
| return (m_ret); | return (m_ret); | |||||||||||
| } | } | |||||||||||
| return (NULL); | return (NULL); | |||||||||||
| } | } | |||||||||||
| /* | /* | |||||||||||
| ▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines | for (segind = vm_phys_nsegs - 1; segind >= 0; segind--) { | |||||||||||
| * we've actually already searched within this | * we've actually already searched within this | |||||||||||
| * too. So skip it. | * too. So skip it. | |||||||||||
| */ | */ | |||||||||||
| if (seg->free_queues == queues) | if (seg->free_queues == queues) | |||||||||||
| continue; | continue; | |||||||||||
| queues = seg->free_queues; | queues = seg->free_queues; | |||||||||||
| m_run = vm_phys_find_queues_contig(queues, npages, | m_run = vm_phys_find_queues_contig(queues, npages, | |||||||||||
| low, high, alignment, boundary); | low, high, alignment, boundary); | |||||||||||
| if (m_run != NULL) | if (m_run != NULL) | |||||||||||
Not Done Inline ActionsThis assertion is redundant. We know that the domain is locked, and we are still operating on the same domain. alc: This assertion is redundant. We know that the domain is locked, and we are still operating on… | ||||||||||||
| break; | break; | |||||||||||
| } | } | |||||||||||
| if (m_run == NULL) | if (m_run == NULL) | |||||||||||
| return (NULL); | return (NULL); | |||||||||||
| /* Allocate pages from the page-range found. */ | /* Allocate pages from the page-range found. */ | |||||||||||
| for (m = m_run; m < &m_run[npages]; m = &m[1 << oind]) { | for (m = m_run; m < &m_run[npages]; m = &m[1 << oind]) { | |||||||||||
| fl = (*queues)[m->pool]; | fl = (*queues)[m->pool]; | |||||||||||
| oind = m->order; | oind = m->order; | |||||||||||
| vm_freelist_rem(fl, m, oind); | vm_freelist_rem(fl, m, oind); | |||||||||||
| if (m->pool != VM_FREEPOOL_DEFAULT) | if (m->pool != VM_FREEPOOL_DEFAULT) | |||||||||||
| vm_phys_set_pool(VM_FREEPOOL_DEFAULT, m, oind); | vm_phys_set_pool(VM_FREEPOOL_DEFAULT, m, oind); | |||||||||||
| } | } | |||||||||||
| /* Return excess pages to the free lists. */ | /* Return excess pages to the free lists. */ | |||||||||||
| fl = (*queues)[VM_FREEPOOL_DEFAULT]; | fl = (*queues)[VM_FREEPOOL_DEFAULT]; | |||||||||||
| vm_phys_enq_range(&m_run[npages], m - &m_run[npages], fl, 0); | vm_phys_enq_range(&m_run[npages], m - &m_run[npages], fl, 0); | |||||||||||
| /* Return page verified to satisfy conditions of request. */ | ||||||||||||
| pa_start = VM_PAGE_TO_PHYS(m_run); | ||||||||||||
| KASSERT(low <= pa_start, | ||||||||||||
| ("memory allocated below minimum requested range")); | ||||||||||||
| KASSERT(pa_start + ptoa(npages) <= high, | ||||||||||||
Not Done Inline Actionsm_run + npages could be outside the segment containing m_run. markj: `m_run + npages` could be outside the segment containing `m_run`. | ||||||||||||
Not Done Inline Actions
alc: | ||||||||||||
| ("memory allocated above maximum requested range")); | ||||||||||||
| seg = &vm_phys_segs[m_run->segind]; | ||||||||||||
Not Done Inline ActionsUse m_run->segind instead to compute this faster. alc: Use m_run->segind instead to compute this faster. | ||||||||||||
| KASSERT(seg->domain == domain, | ||||||||||||
Not Done Inline ActionsThe NULL check is now pointless. alc: The NULL check is now pointless. | ||||||||||||
| ("memory not allocated from specified domain")); | ||||||||||||
| KASSERT(vm_addr_ok(pa_start, ptoa(npages), alignment, boundary), | ||||||||||||
Not Done Inline Actions
alc: | ||||||||||||
| ("memory alignment/boundary contraints not satisfied")); | ||||||||||||
Not Done Inline Actions
markj: | ||||||||||||
| return (m_run); | return (m_run); | |||||||||||
| } | } | |||||||||||
| /* | /* | |||||||||||
| * Return the index of the first unused slot which may be the terminating | * Return the index of the first unused slot which may be the terminating | |||||||||||
| * entry. | * entry. | |||||||||||
| */ | */ | |||||||||||
| static int | static int | |||||||||||
| ▲ Show 20 Lines • Show All 295 Lines • Show Last 20 Lines | ||||||||||||
This is asking for trouble from someone who doesn't carefully read all of the code.