Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/vm_page.c
| Show First 20 Lines • Show All 823 Lines • ▼ Show 20 Lines | |||||
| void | void | ||||
| vm_page_reference(vm_page_t m) | vm_page_reference(vm_page_t m) | ||||
| { | { | ||||
| vm_page_aflag_set(m, PGA_REFERENCED); | vm_page_aflag_set(m, PGA_REFERENCED); | ||||
| } | } | ||||
| /* | |||||
| * vm_page_grab_trybusy | |||||
| * | |||||
| * Helper routine for grab functions to trylock busy. | |||||
| * | |||||
| * Returns true on success and false on failure. | |||||
| */ | |||||
| static bool | static bool | ||||
| vm_page_acquire_flags(vm_page_t m, int allocflags) | vm_page_grab_trybusy(vm_page_t m, int allocflags) | ||||
markj: vm_page_grab_trybusy() is also used by non-grab functions. You could rename it to… | |||||
| { | { | ||||
| bool locked; | |||||
| if ((allocflags & (VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY)) != 0) | if ((allocflags & (VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY)) != 0) | ||||
| locked = vm_page_trysbusy(m); | return (vm_page_trysbusy(m)); | ||||
| else | else | ||||
| locked = vm_page_tryxbusy(m); | return (vm_page_tryxbusy(m)); | ||||
| } | |||||
| /* | |||||
| * vm_page_grab_tryacquire | |||||
| * | |||||
| * Helper routine for grab functions to trylock busy and wire. | |||||
| * | |||||
| * Returns true on success and false on failure. | |||||
| */ | |||||
| static inline bool | |||||
| vm_page_grab_tryacquire(vm_page_t m, int allocflags) | |||||
| { | |||||
| bool locked; | |||||
| locked = vm_page_grab_trybusy(m, allocflags); | |||||
| if (locked && (allocflags & VM_ALLOC_WIRED) != 0) | if (locked && (allocflags & VM_ALLOC_WIRED) != 0) | ||||
| vm_page_wire(m); | vm_page_wire(m); | ||||
| return (locked); | return (locked); | ||||
| } | } | ||||
| /* | /* | ||||
| * vm_page_busy_sleep_flags | * vm_page_grab_release | ||||
| * | * | ||||
| * Helper routine for grab functions to release busy on return. | |||||
| */ | |||||
| static inline void | |||||
| vm_page_grab_release(vm_page_t m, int allocflags) | |||||
| { | |||||
| if ((allocflags & VM_ALLOC_NOBUSY) != 0) { | |||||
| if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0) | |||||
| vm_page_sunbusy(m); | |||||
| else | |||||
| vm_page_xunbusy(m); | |||||
| } | |||||
| } | |||||
| /* | |||||
| * vm_page_grab_sleep | |||||
| * | |||||
| * Sleep for busy according to VM_ALLOC_ parameters. Returns true | * Sleep for busy according to VM_ALLOC_ parameters. Returns true | ||||
| * if the caller should retry and false otherwise. | * if the caller should retry and false otherwise. | ||||
Not Done Inline ActionsAlthough it is obvious for careful reader that the function drops the object lock, it might be surprising to not so careful one. Also we traditionally point that out. So I think it is useful to note that the function might drop the object lock if owned. kib: Although it is obvious for careful reader that the function drops the object lock, it might be… | |||||
| */ | */ | ||||
| static bool | static bool | ||||
| vm_page_busy_sleep_flags(vm_object_t object, vm_page_t m, const char *wmesg, | vm_page_grab_sleep(vm_object_t object, vm_page_t m, vm_pindex_t pindex, | ||||
| int allocflags) | const char *wmesg, int allocflags, bool locked) | ||||
| { | { | ||||
| if ((allocflags & VM_ALLOC_NOWAIT) != 0) | if ((allocflags & VM_ALLOC_NOWAIT) != 0) | ||||
| return (false); | return (false); | ||||
| /* | /* | ||||
| * Reference the page before unlocking and sleeping so that | * Reference the page before unlocking and sleeping so that | ||||
| * the page daemon is less likely to reclaim it. | * the page daemon is less likely to reclaim it. | ||||
| */ | */ | ||||
| if ((allocflags & VM_ALLOC_NOCREAT) == 0) | if (locked && (allocflags & VM_ALLOC_NOCREAT) == 0) | ||||
| vm_page_reference(m); | vm_page_reference(m); | ||||
| if (_vm_page_busy_sleep(object, m, m->pindex, wmesg, allocflags, true)) | if (_vm_page_busy_sleep(object, m, m->pindex, wmesg, allocflags, | ||||
| locked) && locked) | |||||
| VM_OBJECT_WLOCK(object); | VM_OBJECT_WLOCK(object); | ||||
| if ((allocflags & VM_ALLOC_WAITFAIL) != 0) | if ((allocflags & VM_ALLOC_WAITFAIL) != 0) | ||||
| return (false); | return (false); | ||||
| return (true); | return (true); | ||||
| } | } | ||||
| /* | /* | ||||
| Show All 12 Lines | vm_page_busy_acquire(vm_page_t m, int allocflags) | ||||
| * The page-specific object must be cached because page | * The page-specific object must be cached because page | ||||
| * identity can change during the sleep, causing the | * identity can change during the sleep, causing the | ||||
| * re-lock of a different object. | * re-lock of a different object. | ||||
| * It is assumed that a reference to the object is already | * It is assumed that a reference to the object is already | ||||
| * held by the callers. | * held by the callers. | ||||
| */ | */ | ||||
| obj = m->object; | obj = m->object; | ||||
| for (;;) { | for (;;) { | ||||
| if (vm_page_acquire_flags(m, allocflags)) | if (vm_page_grab_tryacquire(m, allocflags)) | ||||
Not Done Inline ActionsWhy tryacquire instead of trybusy? I don't think anything should be using this function to wire the page. If not, then we should rename the function to e.g., vm_page_acquire(). markj: Why tryacquire instead of trybusy? I don't think anything should be using this function to wire… | |||||
Done Inline ActionsI found no reason to preclude wiring the page although it would be unusual. jeff: I found no reason to preclude wiring the page although it would be unusual. | |||||
| return (true); | return (true); | ||||
| if ((allocflags & VM_ALLOC_NOWAIT) != 0) | if ((allocflags & VM_ALLOC_NOWAIT) != 0) | ||||
| return (false); | return (false); | ||||
| if (obj != NULL) | if (obj != NULL) | ||||
| locked = VM_OBJECT_WOWNED(obj); | locked = VM_OBJECT_WOWNED(obj); | ||||
| else | else | ||||
| locked = false; | locked = false; | ||||
| MPASS(locked || vm_page_wired(m)); | MPASS(locked || vm_page_wired(m)); | ||||
| ▲ Show 20 Lines • Show All 698 Lines • ▼ Show 20 Lines | vm_page_object_remove(vm_page_t m) | ||||
| if ((m->a.flags & PGA_SWAP_FREE) != 0) | if ((m->a.flags & PGA_SWAP_FREE) != 0) | ||||
| vm_pager_page_unswapped(m); | vm_pager_page_unswapped(m); | ||||
| mrem = vm_radix_remove(&object->rtree, m->pindex); | mrem = vm_radix_remove(&object->rtree, m->pindex); | ||||
| KASSERT(mrem == m, ("removed page %p, expected page %p", mrem, m)); | KASSERT(mrem == m, ("removed page %p, expected page %p", mrem, m)); | ||||
| /* | /* | ||||
| * Now remove from the object's list of backed pages. | * Now remove from the object's list of backed pages. | ||||
| */ | */ | ||||
Not Done Inline ActionsIs this comment still relevant ? I believe it described a release store. kib: Is this comment still relevant ? I believe it described a release store.
Right now there is… | |||||
Done Inline ActionsThe release store is in vm_radix_remove(). The acquire is in vm_radix_lookup_unlocked(). jeff: The release store is in vm_radix_remove(). The acquire is in vm_radix_lookup_unlocked(). | |||||
| TAILQ_REMOVE(&object->memq, m, listq); | TAILQ_REMOVE(&object->memq, m, listq); | ||||
| /* | /* | ||||
| * And show that the object has one fewer resident page. | * And show that the object has one fewer resident page. | ||||
| */ | */ | ||||
| object->resident_page_count--; | object->resident_page_count--; | ||||
| /* | /* | ||||
| ▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Lines | |||||
| vm_page_lookup(vm_object_t object, vm_pindex_t pindex) | vm_page_lookup(vm_object_t object, vm_pindex_t pindex) | ||||
| { | { | ||||
| VM_OBJECT_ASSERT_LOCKED(object); | VM_OBJECT_ASSERT_LOCKED(object); | ||||
| return (vm_radix_lookup(&object->rtree, pindex)); | return (vm_radix_lookup(&object->rtree, pindex)); | ||||
| } | } | ||||
| /* | /* | ||||
| * This should only be used by lockless functions for releasing transient | |||||
| * incorrect acquires. The page may have been freed after we acquired a | |||||
| * busy lock. In this case busy_lock == VPB_FREED and we have nothing | |||||
| * further to do. | |||||
| */ | |||||
| static void | |||||
| vm_page_busy_release(vm_page_t m) | |||||
| { | |||||
| u_int x; | |||||
| x = atomic_load_int(&m->busy_lock); | |||||
Not Done Inline ActionsThis should be an atomic load. markj: This should be an atomic load. | |||||
Done Inline ActionsFor the reader or the machine? jeff: For the reader or the machine? | |||||
Not Done Inline ActionsFor compiler. Atomic_load() does not add anything to the generated load, except it ensures that the load indeed happen (but could be reordered or coalesced). I think it would be somewhat clearer to see atomic_load() for lock-kind word, as well. kib: For compiler. Atomic_load() does not add anything to the generated load, except it ensures… | |||||
| for (;;) { | |||||
| if (x == VPB_FREED) | |||||
| break; | |||||
| if ((x & VPB_BIT_SHARED) != 0 && VPB_SHARERS(x) > 1) { | |||||
| if (atomic_fcmpset_int(&m->busy_lock, &x, | |||||
| x - VPB_ONE_SHARER)) | |||||
| break; | |||||
Not Done Inline ActionsDon't we potentially need to wake up waiters if we are the last sharer? markj: Don't we potentially need to wake up waiters if we are the last sharer? | |||||
Done Inline ActionsIf we are the last sharer we should execute the case below. jeff: If we are the last sharer we should execute the case below. | |||||
| continue; | |||||
| } | |||||
| KASSERT((x & VPB_BIT_SHARED) != 0 || | |||||
Not Done Inline ActionsShould we assert that x is exclusive busy owner ? kib: Should we assert that x is exclusive busy owner ? | |||||
| (x & ~VPB_BIT_WAITERS) == VPB_CURTHREAD_EXCLUSIVE, | |||||
| ("vm_page_busy_release: %p xbusy not owned.", m)); | |||||
| if (!atomic_fcmpset_rel_int(&m->busy_lock, &x, VPB_UNBUSIED)) | |||||
| continue; | |||||
| if ((x & VPB_BIT_WAITERS) != 0) | |||||
| wakeup(m); | |||||
| break; | |||||
| } | |||||
| } | |||||
| /* | |||||
| * vm_page_find_least: | * vm_page_find_least: | ||||
| * | * | ||||
| * Returns the page associated with the object with least pindex | * Returns the page associated with the object with least pindex | ||||
| * greater than or equal to the parameter pindex, or NULL. | * greater than or equal to the parameter pindex, or NULL. | ||||
| * | * | ||||
| * The object must be locked. | * The object must be locked. | ||||
| */ | */ | ||||
| vm_page_t | vm_page_t | ||||
| ▲ Show 20 Lines • Show All 2,550 Lines • ▼ Show 20 Lines | vm_page_advise(vm_page_t m, int advice) | ||||
| * laundry are moved there. | * laundry are moved there. | ||||
| */ | */ | ||||
| if (m->dirty == 0) | if (m->dirty == 0) | ||||
| vm_page_deactivate_noreuse(m); | vm_page_deactivate_noreuse(m); | ||||
| else if (!vm_page_in_laundry(m)) | else if (!vm_page_in_laundry(m)) | ||||
| vm_page_launder(m); | vm_page_launder(m); | ||||
| } | } | ||||
| static inline int | /* | ||||
| vm_page_grab_pflags(int allocflags) | * Assert that the grab flags are valid. | ||||
| */ | |||||
| static inline void | |||||
| vm_page_grab_check(int allocflags) | |||||
Not Done Inline Actionsvm_page_grab_check() might be a better name. It is consistent with vm_page_alloc_check() and vm_page_grab_asserts sounds like "asserts" is the object, not "vm_page". markj: vm_page_grab_check() might be a better name. It is consistent with vm_page_alloc_check() and… | |||||
Done Inline Actionsok. The naming I really don't like is _unlocked after everything. I could probably do a sweep of the tree and convert everything to _unlocked and drop it from the name. Any outside sources doing direct object manipulation will suddenly be wrong though. If you have suggestions I'm all ears. jeff: ok.
The naming I really don't like is _unlocked after everything. I could probably do a sweep… | |||||
| { | { | ||||
| int pflags; | |||||
| KASSERT((allocflags & VM_ALLOC_NOBUSY) == 0 || | KASSERT((allocflags & VM_ALLOC_NOBUSY) == 0 || | ||||
| (allocflags & VM_ALLOC_WIRED) != 0, | (allocflags & VM_ALLOC_WIRED) != 0, | ||||
| ("vm_page_grab_pflags: the pages must be busied or wired")); | ("vm_page_grab*: the pages must be busied or wired")); | ||||
| KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 || | KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 || | ||||
| (allocflags & VM_ALLOC_IGN_SBUSY) != 0, | (allocflags & VM_ALLOC_IGN_SBUSY) != 0, | ||||
| ("vm_page_grab_pflags: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY " | ("vm_page_grab*: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY mismatch")); | ||||
| "mismatch")); | } | ||||
| /* | |||||
| * Calculate the page allocation flags for grab. | |||||
| */ | |||||
| static inline int | |||||
| vm_page_grab_pflags(int allocflags) | |||||
| { | |||||
| int pflags; | |||||
Not Done Inline ActionsYou can get rid of the extra quotes here. markj: You can get rid of the extra quotes here. | |||||
| pflags = allocflags & | pflags = allocflags & | ||||
| ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL | | ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL | | ||||
| VM_ALLOC_NOBUSY); | VM_ALLOC_NOBUSY); | ||||
| if ((allocflags & VM_ALLOC_NOWAIT) == 0) | if ((allocflags & VM_ALLOC_NOWAIT) == 0) | ||||
| pflags |= VM_ALLOC_WAITFAIL; | pflags |= VM_ALLOC_WAITFAIL; | ||||
| if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0) | if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0) | ||||
| pflags |= VM_ALLOC_SBUSY; | pflags |= VM_ALLOC_SBUSY; | ||||
| Show All 10 Lines | |||||
| * | * | ||||
| * The object must be locked on entry. The lock will, however, be released | * The object must be locked on entry. The lock will, however, be released | ||||
| * and reacquired if the routine sleeps. | * and reacquired if the routine sleeps. | ||||
| */ | */ | ||||
| vm_page_t | vm_page_t | ||||
| vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags) | vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags) | ||||
| { | { | ||||
| vm_page_t m; | vm_page_t m; | ||||
| int pflags; | |||||
| VM_OBJECT_ASSERT_WLOCKED(object); | VM_OBJECT_ASSERT_WLOCKED(object); | ||||
| pflags = vm_page_grab_pflags(allocflags); | vm_page_grab_check(allocflags); | ||||
| retrylookup: | retrylookup: | ||||
| if ((m = vm_page_lookup(object, pindex)) != NULL) { | if ((m = vm_page_lookup(object, pindex)) != NULL) { | ||||
| if (!vm_page_acquire_flags(m, allocflags)) { | if (!vm_page_grab_tryacquire(m, allocflags)) { | ||||
| if (vm_page_busy_sleep_flags(object, m, "pgrbwt", | if (vm_page_grab_sleep(object, m, pindex, "pgrbwt", | ||||
| allocflags)) | allocflags, true)) | ||||
| goto retrylookup; | goto retrylookup; | ||||
| return (NULL); | return (NULL); | ||||
| } | } | ||||
| goto out; | goto out; | ||||
| } | } | ||||
| if ((allocflags & VM_ALLOC_NOCREAT) != 0) | if ((allocflags & VM_ALLOC_NOCREAT) != 0) | ||||
| return (NULL); | return (NULL); | ||||
| m = vm_page_alloc(object, pindex, pflags); | m = vm_page_alloc(object, pindex, vm_page_grab_pflags(allocflags)); | ||||
| if (m == NULL) { | if (m == NULL) { | ||||
| if ((allocflags & (VM_ALLOC_NOWAIT | VM_ALLOC_WAITFAIL)) != 0) | if ((allocflags & (VM_ALLOC_NOWAIT | VM_ALLOC_WAITFAIL)) != 0) | ||||
| return (NULL); | return (NULL); | ||||
| goto retrylookup; | goto retrylookup; | ||||
| } | } | ||||
| if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0) | if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0) | ||||
| pmap_zero_page(m); | pmap_zero_page(m); | ||||
| out: | out: | ||||
| if ((allocflags & VM_ALLOC_NOBUSY) != 0) { | vm_page_grab_release(m, allocflags); | ||||
| if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0) | |||||
| vm_page_sunbusy(m); | return (m); | ||||
| else | |||||
| vm_page_xunbusy(m); | |||||
| } | } | ||||
| /* | |||||
| * Try to locklessly grab a page and fall back to the object lock if NOCREAT | |||||
| * is not set. | |||||
| */ | |||||
| vm_page_t | |||||
| vm_page_grab_unlocked(vm_object_t object, vm_pindex_t pindex, int allocflags) | |||||
| { | |||||
| vm_page_t m; | |||||
| vm_page_grab_check(allocflags); | |||||
| for (;;) { | |||||
| m = vm_radix_lookup_unlocked(&object->rtree, pindex); | |||||
| if (m == NULL) | |||||
| break; | |||||
| if (!vm_page_grab_trybusy(m, allocflags)) { | |||||
| if (vm_page_grab_sleep(object, m, pindex, "pgrbwt", | |||||
| allocflags, false)) | |||||
| continue; | |||||
| return (NULL); | |||||
| } | |||||
| if (m->object != object || m->pindex != pindex) { | |||||
| vm_page_busy_release(m); | |||||
| continue; | |||||
| } | |||||
| if ((allocflags & VM_ALLOC_WIRED) != 0) | |||||
| vm_page_wire(m); | |||||
| vm_page_grab_release(m, allocflags); | |||||
| return (m); | return (m); | ||||
| } | } | ||||
| /* | /* | ||||
| * The radix lockless lookup should never return spurious errors. If | |||||
| * the user specifies NOCREAT they are guaranteed there was no page | |||||
Not Done Inline ActionsThe use of TAILQ_NEXT here is technically fine but breaks the locking protocol. In the absence of an "atomic" TAILQ_NEXT it would probably be good to note this in the comment. markj: The use of TAILQ_NEXT here is technically fine but breaks the locking protocol. In the absence… | |||||
Done Inline ActionsThat was the intent of the comment but I can clarify. jeff: That was the intent of the comment but I can clarify. | |||||
| * present at the instant of the call. A NOCREAT caller must handle | |||||
| * create races gracefully. | |||||
| */ | |||||
| if ((allocflags & VM_ALLOC_NOCREAT) != 0) | |||||
| return (NULL); | |||||
| VM_OBJECT_WLOCK(object); | |||||
| m = vm_page_grab(object, pindex, allocflags); | |||||
| VM_OBJECT_WUNLOCK(object); | |||||
| return (m); | |||||
| } | |||||
Not Done Inline ActionsMaybe use __predict_true here. markj: Maybe use __predict_true here. | |||||
| /* | |||||
| * Grab a page and make it valid, paging in if necessary. Pages missing from | * Grab a page and make it valid, paging in if necessary. Pages missing from | ||||
| * their pager are zero filled and validated. If a VM_ALLOC_COUNT is supplied | * their pager are zero filled and validated. If a VM_ALLOC_COUNT is supplied | ||||
| * and the page is not valid as many as VM_INITIAL_PAGEIN pages can be brought | * and the page is not valid as many as VM_INITIAL_PAGEIN pages can be brought | ||||
| * in simultaneously. Additional pages will be left on a paging queue but | * in simultaneously. Additional pages will be left on a paging queue but | ||||
| * will neither be wired nor busy regardless of allocflags. | * will neither be wired nor busy regardless of allocflags. | ||||
| */ | */ | ||||
| int | int | ||||
Not Done Inline Actions"Spurious errors" is kind of ambiguous. "False negatives"? markj: "Spurious errors" is kind of ambiguous. "False negatives"? | |||||
| vm_page_grab_valid(vm_page_t *mp, vm_object_t object, vm_pindex_t pindex, int allocflags) | vm_page_grab_valid(vm_page_t *mp, vm_object_t object, vm_pindex_t pindex, int allocflags) | ||||
| { | { | ||||
| vm_page_t m; | vm_page_t m; | ||||
| vm_page_t ma[VM_INITIAL_PAGEIN]; | vm_page_t ma[VM_INITIAL_PAGEIN]; | ||||
| bool sleep, xbusy; | |||||
| int after, i, pflags, rv; | int after, i, pflags, rv; | ||||
| KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 || | KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 || | ||||
| (allocflags & VM_ALLOC_IGN_SBUSY) != 0, | (allocflags & VM_ALLOC_IGN_SBUSY) != 0, | ||||
| ("vm_page_grab_valid: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY mismatch")); | ("vm_page_grab_valid: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY mismatch")); | ||||
| KASSERT((allocflags & | KASSERT((allocflags & | ||||
| (VM_ALLOC_NOWAIT | VM_ALLOC_WAITFAIL | VM_ALLOC_ZERO)) == 0, | (VM_ALLOC_NOWAIT | VM_ALLOC_WAITFAIL | VM_ALLOC_ZERO)) == 0, | ||||
| ("vm_page_grab_valid: Invalid flags 0x%X", allocflags)); | ("vm_page_grab_valid: Invalid flags 0x%X", allocflags)); | ||||
| VM_OBJECT_ASSERT_WLOCKED(object); | VM_OBJECT_ASSERT_WLOCKED(object); | ||||
| pflags = allocflags & ~(VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY); | pflags = allocflags & ~(VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY); | ||||
| pflags |= VM_ALLOC_WAITFAIL; | pflags |= VM_ALLOC_WAITFAIL; | ||||
| retrylookup: | retrylookup: | ||||
| xbusy = false; | |||||
| if ((m = vm_page_lookup(object, pindex)) != NULL) { | if ((m = vm_page_lookup(object, pindex)) != NULL) { | ||||
| /* | /* | ||||
| * If the page is fully valid it can only become invalid | * If the page is fully valid it can only become invalid | ||||
| * with the object lock held. If it is not valid it can | * with the object lock held. If it is not valid it can | ||||
| * become valid with the busy lock held. Therefore, we | * become valid with the busy lock held. Therefore, we | ||||
| * may unnecessarily lock the exclusive busy here if we | * may unnecessarily lock the exclusive busy here if we | ||||
| * race with I/O completion not using the object lock. | * race with I/O completion not using the object lock. | ||||
| * However, we will not end up with an invalid page and a | * However, we will not end up with an invalid page and a | ||||
| * shared lock. | * shared lock. | ||||
| */ | */ | ||||
| if (!vm_page_all_valid(m) || | if (!vm_page_grab_trybusy(m, | ||||
| (allocflags & (VM_ALLOC_IGN_SBUSY | VM_ALLOC_SBUSY)) == 0) { | vm_page_all_valid(m) ? allocflags : 0)) { | ||||
| sleep = !vm_page_tryxbusy(m); | (void)vm_page_grab_sleep(object, m, pindex, "pgrbwt", | ||||
| xbusy = true; | allocflags, true); | ||||
| } else | |||||
| sleep = !vm_page_trysbusy(m); | |||||
| if (sleep) { | |||||
| (void)vm_page_busy_sleep_flags(object, m, "pgrbwt", | |||||
| allocflags); | |||||
| goto retrylookup; | goto retrylookup; | ||||
| } | } | ||||
| if ((allocflags & VM_ALLOC_NOCREAT) != 0 && | if (vm_page_all_valid(m)) | ||||
| !vm_page_all_valid(m)) { | goto out; | ||||
| if (xbusy) | if ((allocflags & VM_ALLOC_NOCREAT) != 0) { | ||||
| vm_page_xunbusy(m); | vm_page_busy_release(m); | ||||
| else | |||||
| vm_page_sunbusy(m); | |||||
| *mp = NULL; | *mp = NULL; | ||||
| return (VM_PAGER_FAIL); | return (VM_PAGER_FAIL); | ||||
| } | } | ||||
| if ((allocflags & VM_ALLOC_WIRED) != 0) | |||||
| vm_page_wire(m); | |||||
| if (vm_page_all_valid(m)) | |||||
| goto out; | |||||
| } else if ((allocflags & VM_ALLOC_NOCREAT) != 0) { | } else if ((allocflags & VM_ALLOC_NOCREAT) != 0) { | ||||
| *mp = NULL; | *mp = NULL; | ||||
| return (VM_PAGER_FAIL); | return (VM_PAGER_FAIL); | ||||
| } else if ((m = vm_page_alloc(object, pindex, pflags)) != NULL) { | } else if ((m = vm_page_alloc(object, pindex, pflags)) == NULL) { | ||||
| xbusy = true; | |||||
| } else { | |||||
| goto retrylookup; | goto retrylookup; | ||||
| } | } | ||||
| vm_page_assert_xbusied(m); | vm_page_assert_xbusied(m); | ||||
| MPASS(xbusy); | |||||
| if (vm_pager_has_page(object, pindex, NULL, &after)) { | if (vm_pager_has_page(object, pindex, NULL, &after)) { | ||||
| after = MIN(after, VM_INITIAL_PAGEIN); | after = MIN(after, VM_INITIAL_PAGEIN); | ||||
| after = MIN(after, allocflags >> VM_ALLOC_COUNT_SHIFT); | after = MIN(after, allocflags >> VM_ALLOC_COUNT_SHIFT); | ||||
| after = MAX(after, 1); | after = MAX(after, 1); | ||||
| ma[0] = m; | ma[0] = m; | ||||
| for (i = 1; i < after; i++) { | for (i = 1; i < after; i++) { | ||||
| if ((ma[i] = vm_page_next(ma[i - 1])) != NULL) { | if ((ma[i] = vm_page_next(ma[i - 1])) != NULL) { | ||||
| if (ma[i]->valid || !vm_page_tryxbusy(ma[i])) | if (ma[i]->valid || !vm_page_tryxbusy(ma[i])) | ||||
| Show All 9 Lines | if (vm_pager_has_page(object, pindex, NULL, &after)) { | ||||
| vm_object_pip_add(object, after); | vm_object_pip_add(object, after); | ||||
| VM_OBJECT_WUNLOCK(object); | VM_OBJECT_WUNLOCK(object); | ||||
| rv = vm_pager_get_pages(object, ma, after, NULL, NULL); | rv = vm_pager_get_pages(object, ma, after, NULL, NULL); | ||||
| VM_OBJECT_WLOCK(object); | VM_OBJECT_WLOCK(object); | ||||
| vm_object_pip_wakeupn(object, after); | vm_object_pip_wakeupn(object, after); | ||||
| /* Pager may have replaced a page. */ | /* Pager may have replaced a page. */ | ||||
| m = ma[0]; | m = ma[0]; | ||||
| if (rv != VM_PAGER_OK) { | if (rv != VM_PAGER_OK) { | ||||
| if ((allocflags & VM_ALLOC_WIRED) != 0) | |||||
| vm_page_unwire_noq(m); | |||||
| for (i = 0; i < after; i++) { | for (i = 0; i < after; i++) { | ||||
| if (!vm_page_wired(ma[i])) | if (!vm_page_wired(ma[i])) | ||||
| vm_page_free(ma[i]); | vm_page_free(ma[i]); | ||||
| else | else | ||||
| vm_page_xunbusy(ma[i]); | vm_page_xunbusy(ma[i]); | ||||
| } | } | ||||
| *mp = NULL; | *mp = NULL; | ||||
| return (rv); | return (rv); | ||||
| } | } | ||||
| for (i = 1; i < after; i++) | for (i = 1; i < after; i++) | ||||
| vm_page_readahead_finish(ma[i]); | vm_page_readahead_finish(ma[i]); | ||||
| MPASS(vm_page_all_valid(m)); | MPASS(vm_page_all_valid(m)); | ||||
| } else { | } else { | ||||
| vm_page_zero_invalid(m, TRUE); | vm_page_zero_invalid(m, TRUE); | ||||
| } | } | ||||
| out: | out: | ||||
| if ((allocflags & VM_ALLOC_NOBUSY) != 0) { | if ((allocflags & VM_ALLOC_WIRED) != 0) | ||||
| if (xbusy) | vm_page_wire(m); | ||||
| vm_page_xunbusy(m); | if ((allocflags & VM_ALLOC_SBUSY) != 0 && vm_page_xbusied(m)) | ||||
| vm_page_busy_downgrade(m); | |||||
| else if ((allocflags & VM_ALLOC_NOBUSY) != 0) | |||||
| vm_page_busy_release(m); | |||||
| *mp = m; | |||||
| return (VM_PAGER_OK); | |||||
| } | |||||
| int | |||||
| vm_page_grab_valid_unlocked(vm_page_t *mp, vm_object_t object, | |||||
| vm_pindex_t pindex, int allocflags) | |||||
| { | |||||
| vm_page_t m; | |||||
| int flags; | |||||
| int error; | |||||
| /* | |||||
| * Attempt a lockless lookup and busy. We need at least an sbusy | |||||
| * before we can inspect the valid field and return a wired page. | |||||
| */ | |||||
| if (allocflags & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) | |||||
| flags = VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY | VM_ALLOC_NOCREAT; | |||||
| else | else | ||||
| flags = VM_ALLOC_NOCREAT; | |||||
| m = vm_page_grab_unlocked(object, pindex, flags); | |||||
| if (m != NULL) { | |||||
| if (vm_page_all_valid(m)) { | |||||
| if ((allocflags & VM_ALLOC_WIRED) != 0) | |||||
| vm_page_wire(m); | |||||
| if ((allocflags & VM_ALLOC_NOBUSY) != 0) | |||||
| vm_page_sunbusy(m); | vm_page_sunbusy(m); | ||||
| } | |||||
| if ((allocflags & VM_ALLOC_SBUSY) != 0 && xbusy) | |||||
| vm_page_busy_downgrade(m); | |||||
| *mp = m; | *mp = m; | ||||
| return (VM_PAGER_OK); | return (VM_PAGER_OK); | ||||
| } | } | ||||
| vm_page_busy_release(m); | |||||
| } | |||||
| VM_OBJECT_WLOCK(object); | |||||
| error = vm_page_grab_valid(mp, object, pindex, allocflags); | |||||
| VM_OBJECT_WUNLOCK(object); | |||||
Not Done Inline ActionsYou might add a two line vm_page_grab_wire() for this, since it gets copied around a lot. markj: You might add a two line vm_page_grab_wire() for this, since it gets copied around a lot. | |||||
Done Inline ActionsThis and _valid() don't want to do it via _acquire or tryacquire because they may decide not to keep the page and this simplified error handling. So they should be the only two IIRC. jeff: This and _valid() don't want to do it via _acquire or tryacquire because they may decide not to… | |||||
| return (error); | |||||
| } | |||||
| /* | /* | ||||
| * Locklessly attempt to acquire a pindex sequential page. | |||||
| * | |||||
| * This routine will not sleep if the page busy lock is acquired. Callers | |||||
| * are responsible for retrying under the object lock. | |||||
| */ | |||||
| static vm_page_t | |||||
| vm_page_grab_next(vm_page_t m, int allocflags) | |||||
| { | |||||
| vm_object_t object; | |||||
| vm_page_t next; | |||||
| vm_pindex_t pindex; | |||||
| vm_page_grab_check(allocflags); | |||||
| MPASS(vm_page_busied(m) || vm_page_wired(m)); | |||||
markjUnsubmitted Not Done Inline ActionsWhy is it sufficient to hold a wiring? Without the object lock or page busied the page's <obj, pindex> identity is not stable. markj: Why is it sufficient to hold a wiring? Without the object lock or page busied the page's <obj… | |||||
jeffAuthorUnsubmitted Done Inline ActionsI think this is actually harmless because the page in question will have a NULL object pointer and its next pointer will either be clear or point to a page without a NULL object pointer. pindex will remain unchanged. I suppose that could create a spin loop until the NULL pointer makes it to the list. So from that pov I need to be more careful here and perhaps retry less. Another option would be to pass the expected object pointer in with the page for validation. Once grab_next fails grab_pages_unlocked will fall back to the locked loop. So the subsequent pages will be valid. I don't think it's possible to get incorrect results this way. jeff: I think this is actually harmless because the page in question will have a NULL object pointer… | |||||
| object = m->object; | |||||
| pindex = m->pindex + 1; | |||||
| for (;;) { | |||||
| /* | |||||
| * We may get spurious errors here because the list is not | |||||
| * modified with barriers. This makes it unsafe to return | |||||
| * a consistent result. | |||||
| */ | |||||
| next = TAILQ_NEXT(m, listq); | |||||
| if (next == NULL || next->pindex != pindex) | |||||
| break; | |||||
| if (!vm_page_grab_trybusy(next, allocflags)) | |||||
| break; | |||||
| if (next->object == object && next->pindex == pindex) { | |||||
| if ((allocflags & VM_ALLOC_WIRED) != 0) | |||||
| vm_page_wire(next); | |||||
| vm_page_grab_release(next, allocflags); | |||||
| return (next); | |||||
| } | |||||
| vm_page_busy_release(next); | |||||
| cpu_spinwait(); | |||||
| } | |||||
| return (NULL); | |||||
| } | |||||
| /* | |||||
| * Return the specified range of pages from the given object. For each | * Return the specified range of pages from the given object. For each | ||||
| * page offset within the range, if a page already exists within the object | * page offset within the range, if a page already exists within the object | ||||
| * at that offset and it is busy, then wait for it to change state. If, | * at that offset and it is busy, then wait for it to change state. If, | ||||
| * instead, the page doesn't exist, then allocate it. | * instead, the page doesn't exist, then allocate it. | ||||
| * | * | ||||
| * The caller must always specify an allocation class. | * The caller must always specify an allocation class. | ||||
| * | * | ||||
| * allocation classes: | * allocation classes: | ||||
Done Inline ActionsWe could actually handle more cases by falling back to a lockless lookup. Then the grab_next_unlocked return would be conclusive. jeff: We could actually handle more cases by falling back to a lockless lookup. Then the… | |||||
| * VM_ALLOC_NORMAL normal process request | * VM_ALLOC_NORMAL normal process request | ||||
| * VM_ALLOC_SYSTEM system *really* needs the pages | * VM_ALLOC_SYSTEM system *really* needs the pages | ||||
| * | * | ||||
| * The caller must always specify that the pages are to be busied and/or | * The caller must always specify that the pages are to be busied and/or | ||||
| * wired. | * wired. | ||||
| * | * | ||||
| * optional allocation flags: | * optional allocation flags: | ||||
| * VM_ALLOC_IGN_SBUSY do not sleep on soft busy pages | * VM_ALLOC_IGN_SBUSY do not sleep on soft busy pages | ||||
| Show All 12 Lines | |||||
| { | { | ||||
| vm_page_t m, mpred; | vm_page_t m, mpred; | ||||
| int pflags; | int pflags; | ||||
| int i; | int i; | ||||
| VM_OBJECT_ASSERT_WLOCKED(object); | VM_OBJECT_ASSERT_WLOCKED(object); | ||||
| KASSERT(((u_int)allocflags >> VM_ALLOC_COUNT_SHIFT) == 0, | KASSERT(((u_int)allocflags >> VM_ALLOC_COUNT_SHIFT) == 0, | ||||
| ("vm_page_grap_pages: VM_ALLOC_COUNT() is not allowed")); | ("vm_page_grap_pages: VM_ALLOC_COUNT() is not allowed")); | ||||
| vm_page_grab_check(allocflags); | |||||
| pflags = vm_page_grab_pflags(allocflags); | pflags = vm_page_grab_pflags(allocflags); | ||||
| if (count == 0) | if (count == 0) | ||||
| return (0); | return (0); | ||||
| i = 0; | i = 0; | ||||
| retrylookup: | retrylookup: | ||||
| m = vm_radix_lookup_le(&object->rtree, pindex + i); | m = vm_radix_lookup_le(&object->rtree, pindex + i); | ||||
| if (m == NULL || m->pindex != pindex + i) { | if (m == NULL || m->pindex != pindex + i) { | ||||
| mpred = m; | mpred = m; | ||||
| m = NULL; | m = NULL; | ||||
| } else | } else | ||||
| mpred = TAILQ_PREV(m, pglist, listq); | mpred = TAILQ_PREV(m, pglist, listq); | ||||
| for (; i < count; i++) { | for (; i < count; i++) { | ||||
| if (m != NULL) { | if (m != NULL) { | ||||
| if (!vm_page_acquire_flags(m, allocflags)) { | if (!vm_page_grab_tryacquire(m, allocflags)) { | ||||
| if (vm_page_busy_sleep_flags(object, m, | if (vm_page_grab_sleep(object, m, pindex, | ||||
| "grbmaw", allocflags)) | "grbmaw", allocflags, true)) | ||||
| goto retrylookup; | goto retrylookup; | ||||
| break; | break; | ||||
| } | } | ||||
| } else { | } else { | ||||
| if ((allocflags & VM_ALLOC_NOCREAT) != 0) | if ((allocflags & VM_ALLOC_NOCREAT) != 0) | ||||
| break; | break; | ||||
| m = vm_page_alloc_after(object, pindex + i, | m = vm_page_alloc_after(object, pindex + i, | ||||
| pflags | VM_ALLOC_COUNT(count - i), mpred); | pflags | VM_ALLOC_COUNT(count - i), mpred); | ||||
| if (m == NULL) { | if (m == NULL) { | ||||
| if ((allocflags & (VM_ALLOC_NOWAIT | | if ((allocflags & (VM_ALLOC_NOWAIT | | ||||
| VM_ALLOC_WAITFAIL)) != 0) | VM_ALLOC_WAITFAIL)) != 0) | ||||
| break; | break; | ||||
| goto retrylookup; | goto retrylookup; | ||||
| } | } | ||||
| } | } | ||||
| if (vm_page_none_valid(m) && | if (vm_page_none_valid(m) && | ||||
| (allocflags & VM_ALLOC_ZERO) != 0) { | (allocflags & VM_ALLOC_ZERO) != 0) { | ||||
| if ((m->flags & PG_ZERO) == 0) | if ((m->flags & PG_ZERO) == 0) | ||||
| pmap_zero_page(m); | pmap_zero_page(m); | ||||
| vm_page_valid(m); | vm_page_valid(m); | ||||
| } | } | ||||
| if ((allocflags & VM_ALLOC_NOBUSY) != 0) { | vm_page_grab_release(m, allocflags); | ||||
| if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0) | |||||
| vm_page_sunbusy(m); | |||||
| else | |||||
| vm_page_xunbusy(m); | |||||
| } | |||||
| ma[i] = mpred = m; | ma[i] = mpred = m; | ||||
| m = vm_page_next(m); | m = vm_page_next(m); | ||||
| } | |||||
| return (i); | |||||
| } | |||||
| int | |||||
| vm_page_grab_pages_unlocked(vm_object_t object, vm_pindex_t pindex, | |||||
| int allocflags, vm_page_t *ma, int count) | |||||
| { | |||||
| vm_page_t m; | |||||
| int flags; | |||||
| int i; | |||||
| vm_page_grab_check(allocflags); | |||||
| i = 0; | |||||
| /* | |||||
| * Modify flags for lockless grab to fail and fall back to the | |||||
| * locked variant rather than attempting to allocate etc. | |||||
| */ | |||||
| flags = allocflags | VM_ALLOC_NOCREAT; | |||||
| if ((allocflags & VM_ALLOC_NOBUSY) != 0) { | |||||
| flags &= ~VM_ALLOC_NOBUSY; | |||||
| if ((flags & VM_ALLOC_IGN_SBUSY) != 0) | |||||
| flags |= VM_ALLOC_SBUSY; | |||||
Not Done Inline ActionsI think "next" should be called "pred" or "prev". markj: I think "next" should be called "pred" or "prev". | |||||
| } | |||||
| m = vm_page_grab_unlocked(object, pindex, flags); | |||||
| while (m != NULL) { | |||||
| if (vm_page_none_valid(m) && | |||||
| (allocflags & VM_ALLOC_ZERO) != 0) { | |||||
| if ((m->flags & PG_ZERO) == 0) | |||||
| pmap_zero_page(m); | |||||
| vm_page_valid(m); | |||||
| } | |||||
| /* m will still be wired or busy according to flags. */ | |||||
| ma[i] = m; | |||||
| vm_page_grab_release(m, allocflags); | |||||
| if (++i == count) | |||||
| break; | |||||
| m = vm_page_grab_next(m, flags); | |||||
| } | |||||
| if (i < count) { | |||||
| count -= i; | |||||
| pindex += i; | |||||
| VM_OBJECT_WLOCK(object); | |||||
| i += vm_page_grab_pages(object, pindex, allocflags, &ma[i], | |||||
| count); | |||||
| VM_OBJECT_WUNLOCK(object); | |||||
| } | } | ||||
| return (i); | return (i); | ||||
| } | } | ||||
| /* | /* | ||||
| * Mapping function for valid or dirty bits in a page. | * Mapping function for valid or dirty bits in a page. | ||||
| * | * | ||||
| * Inputs are required to range within a page. | * Inputs are required to range within a page. | ||||
| ▲ Show 20 Lines • Show All 669 Lines • Show Last 20 Lines | |||||
vm_page_grab_trybusy() is also used by non-grab functions. You could rename it to vm_page_trybusy() and group it with the other functions in this file that deal with page busying.