Index: sys/vm/vm_map.c =================================================================== --- sys/vm/vm_map.c +++ sys/vm/vm_map.c @@ -993,7 +993,7 @@ /* vm_map_entry_succ is defined in vm_map.h. */ -#define SPLAY_LEFT_STEP(root, y, rlist, test) do { \ +#define SPLAY_LEFT_STEP(root, y, rlist, found, test) do { \ vm_size_t max_free; \ \ /* \ @@ -1024,9 +1024,10 @@ root->left = rlist; \ rlist = root; \ root = y; \ + found = root != NULL; \ } while (0) -#define SPLAY_RIGHT_STEP(root, y, llist, test) do { \ +#define SPLAY_RIGHT_STEP(root, y, llist, found, test) do { \ vm_size_t max_free; \ \ /* \ @@ -1057,6 +1058,7 @@ root->right = llist; \ llist = root; \ root = y; \ + found = root != NULL; \ } while (0) /* @@ -1069,55 +1071,52 @@ * the subsequent call to vm_map_splay_merge, rely on the start and end address * values in &map->header. */ -static vm_map_entry_t -vm_map_splay_split(vm_map_t map, vm_offset_t addr, vm_size_t length, - vm_map_entry_t *out_llist, vm_map_entry_t *out_rlist) -{ - vm_map_entry_t llist, rlist, root, y; +#define VM_MAP_SPLAY_SPLIT(map, addr, length, root, llist, rlist, found)\ +do { \ + vm_map_entry_t y; \ + \ + llist = rlist = &map->header; \ + root = map->root; \ + found = root != NULL; \ + while (found && root->max_free >= length) { \ + KASSERT(llist->end <= root->start && \ + root->end <= rlist->start, \ + ("%s: root not within tree bounds", __func__)); \ + if (addr < root->start) { \ + SPLAY_LEFT_STEP(root, y, rlist, found, \ + y->max_free >= length && addr < y->start); \ + } else if (addr >= root->end) { \ + SPLAY_RIGHT_STEP(root, y, llist, found, \ + y->max_free >= length && addr >= y->end); \ + } else \ + break; \ + } \ +} while (0) - llist = rlist = &map->header; - root = map->root; - while (root != NULL && root->max_free >= length) { - KASSERT(llist->end <= root->start && root->end <= rlist->start, - ("%s: root not within tree bounds", __func__)); - if (addr < root->start) { - SPLAY_LEFT_STEP(root, y, rlist, - y->max_free >= length && addr < y->start); - } else if (addr >= root->end) { - SPLAY_RIGHT_STEP(root, y, llist, - y->max_free >= length && addr >= y->end); - } else - break; - } - *out_llist = llist; - *out_rlist = rlist; - return (root); -} +#define VM_MAP_SPLAY_FINDNEXT(root, rlist) do { \ + vm_map_entry_t hi, y; \ + bool found; \ + \ + hi = root->right; \ + if (hi == NULL) \ + break; \ + do \ + SPLAY_LEFT_STEP(hi, y, rlist, found, true); \ + while (found); \ +} while (0) -static void -vm_map_splay_findnext(vm_map_entry_t root, vm_map_entry_t *iolist) -{ - vm_map_entry_t rlist, y; +#define VM_MAP_SPLAY_FINDPREV(root, llist) do { \ + vm_map_entry_t lo, y; \ + bool found; \ + \ + lo = root->left; \ + if (lo == NULL) \ + break; \ + do \ + SPLAY_RIGHT_STEP(lo, y, llist, found, true); \ + while (found); \ +} while (0) - root = root->right; - rlist = *iolist; - while (root != NULL) - SPLAY_LEFT_STEP(root, y, rlist, true); - *iolist = rlist; -} - -static void -vm_map_splay_findprev(vm_map_entry_t root, vm_map_entry_t *iolist) -{ - vm_map_entry_t llist, y; - - root = root->left; - llist = *iolist; - while (root != NULL) - SPLAY_RIGHT_STEP(root, y, llist, true); - *iolist = llist; -} - static inline void vm_map_entry_swap(vm_map_entry_t *a, vm_map_entry_t *b) { @@ -1199,9 +1198,10 @@ vm_map_splay(vm_map_t map, vm_offset_t addr) { vm_map_entry_t llist, rlist, root; + bool found; - root = vm_map_splay_split(map, addr, 0, &llist, &rlist); - if (root != NULL) { + VM_MAP_SPLAY_SPLIT(map, addr, 0, root, llist, rlist, found); + if (found) { /* do nothing */ } else if (llist != &map->header) { /* @@ -1237,14 +1237,15 @@ vm_map_entry_link(vm_map_t map, vm_map_entry_t entry) { vm_map_entry_t llist, rlist, root; + bool found; CTR3(KTR_VM, "vm_map_entry_link: map %p, nentries %d, entry %p", map, map->nentries, entry); VM_MAP_ASSERT_LOCKED(map); map->nentries++; - root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); - KASSERT(root == NULL, + VM_MAP_SPLAY_SPLIT(map, entry->start, 0, root, llist, rlist, found); + KASSERT(!found, ("vm_map_entry_link: link object already mapped")); entry->prev = llist; entry->next = rlist; @@ -1264,36 +1265,28 @@ enum unlink_merge_type op) { vm_map_entry_t llist, rlist, root, y; + bool found; VM_MAP_ASSERT_LOCKED(map); - root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); - KASSERT(root != NULL, - ("vm_map_entry_unlink: unlink object not mapped")); + VM_MAP_SPLAY_SPLIT(map, entry->start, 0, root, llist, rlist, found); + KASSERT(found, ("vm_map_entry_unlink: unlink object not mapped")); - vm_map_splay_findnext(root, &rlist); - switch (op) { - case UNLINK_MERGE_NEXT: + VM_MAP_SPLAY_FINDPREV(root, llist); + VM_MAP_SPLAY_FINDNEXT(root, rlist); + if (op == UNLINK_MERGE_NEXT) { rlist->start = root->start; rlist->offset = root->offset; - y = root->left; + } + if (llist != &map->header) { + root = llist; + llist = root->right; + root->right = NULL; + } else if (rlist != &map->header) { root = rlist; rlist = root->left; - root->left = y; - break; - case UNLINK_MERGE_NONE: - vm_map_splay_findprev(root, &llist); - if (llist != &map->header) { - root = llist; - llist = root->right; - root->right = NULL; - } else if (rlist != &map->header) { - root = rlist; - rlist = root->left; - root->left = NULL; - } else - root = NULL; - break; - } + root->left = NULL; + } else + root = NULL; y = entry->next; y->prev = entry->prev; y->prev->next = y; @@ -1319,12 +1312,12 @@ vm_map_entry_resize(vm_map_t map, vm_map_entry_t entry, vm_size_t grow_amount) { vm_map_entry_t llist, rlist, root; + bool found; VM_MAP_ASSERT_LOCKED(map); - root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); - KASSERT(root != NULL, - ("%s: resize object not mapped", __func__)); - vm_map_splay_findnext(root, &rlist); + VM_MAP_SPLAY_SPLIT(map, entry->start, 0, root, llist, rlist, found); + KASSERT(found, ("%s: resize object not mapped", __func__)); + VM_MAP_SPLAY_FINDNEXT(root, rlist); root->right = NULL; entry->end += grow_amount; vm_map_splay_merge(map, root, llist, rlist); @@ -1647,7 +1640,7 @@ { vm_map_entry_t llist, rlist, root, y; vm_size_t left_length; - vm_offset_t gap_end; + bool found; /* * Request must fit within min/max VM address and must avoid @@ -1664,16 +1657,14 @@ /* * After splay_split, if start is within an entry, push it to the start * of the following gap. If rlist is at the end of the gap containing - * start, save the end of that gap in gap_end to see if the gap is big - * enough; otherwise set gap_end to start skip gap-checking and move - * directly to a search of the right subtree. + * start, clear 'found' and see if the gap is big enough. */ - root = vm_map_splay_split(map, start, length, &llist, &rlist); - gap_end = rlist->start; - if (root != NULL) { - start = root->end; - if (root->right != NULL) - gap_end = start; + VM_MAP_SPLAY_SPLIT(map, start, length, root, llist, rlist, found); + if (found) { + if (root->right == NULL) { + start = root->end; + found = false; + } } else if (rlist != &map->header) { root = rlist; rlist = root->left; @@ -1685,7 +1676,7 @@ } vm_map_splay_merge(map, root, llist, rlist); VM_MAP_ASSERT_CONSISTENT(map); - if (length <= gap_end - start) + if (!found && length <= rlist->start - start) return (start); /* With max_free, can immediately tell if no solution. */ @@ -1699,12 +1690,12 @@ for (left_length = 0;; left_length = vm_map_entry_max_free_left(root, llist)) { if (length <= left_length) - SPLAY_LEFT_STEP(root, y, rlist, + SPLAY_LEFT_STEP(root, y, rlist, found, length <= vm_map_entry_max_free_left(y, llist)); else - SPLAY_RIGHT_STEP(root, y, llist, + SPLAY_RIGHT_STEP(root, y, llist, found, length > vm_map_entry_max_free_left(y, root)); - if (root == NULL) + if (!found) break; } root = llist;