Index: sys/vm/vm_map.c =================================================================== --- sys/vm/vm_map.c +++ sys/vm/vm_map.c @@ -923,10 +923,12 @@ /* Rotate right and make y root. */ \ root->left = y->right; \ y->right = root; \ - vm_map_entry_set_max_free(root); \ + if (root->max_free == y->max_free) \ + vm_map_entry_set_max_free(root);\ root = y; \ y = root->left; \ } \ + root->max_free = root->right ? root->right->max_free : root->next->start - root->end;\ /* Put root on rlist. */ \ root->left = rlist; \ rlist = root; \ @@ -939,10 +941,12 @@ /* Rotate left and make y root. */ \ root->right = y->left; \ y->left = root; \ - vm_map_entry_set_max_free(root); \ + if (root->max_free == y->max_free) \ + vm_map_entry_set_max_free(root);\ root = y; \ y = root->right; \ } \ + root->max_free = root->left ? root->left->max_free : root->start - root->prev->end;\ /* Put root on llist. */ \ root->right = llist; \ llist = root; \ @@ -959,13 +963,13 @@ */ static vm_map_entry_t vm_map_splay_split(vm_offset_t addr, vm_size_t length, - vm_map_entry_t root, vm_map_entry_t *out_llist, vm_map_entry_t *out_rlist) + vm_map_entry_t root, vm_map_entry_t *io_llist, vm_map_entry_t *io_rlist) { vm_map_entry_t llist, rlist; vm_map_entry_t y; - llist = NULL; - rlist = NULL; + llist = *io_llist; + rlist = *io_rlist; while (root != NULL && root->max_free >= length) { if (addr < root->start) { SPLAY_LEFT_STEP(root, y, rlist, @@ -976,8 +980,8 @@ } else break; } - *out_llist = llist; - *out_rlist = rlist; + *io_llist = llist; + *io_rlist = rlist; return (root); } @@ -1015,18 +1019,31 @@ vm_map_entry_t ltree, vm_map_entry_t rtree) { vm_map_entry_t y; + vm_size_t max_free; + if (llist != NULL) + max_free = (ltree != NULL) ? ltree->max_free : + llist->next->start - llist->end; while (llist != NULL) { y = llist->right; llist->right = ltree; - vm_map_entry_set_max_free(llist); + if (llist->max_free < max_free) + llist->max_free = max_free; + else + max_free = llist->max_free; ltree = llist; llist = y; } + if (rlist != NULL) + max_free = (rtree != NULL) ? rtree->max_free : + rlist->start - rlist->prev->end; while (rlist != NULL) { y = rlist->left; rlist->left = rtree; - vm_map_entry_set_max_free(rlist); + if (rlist->max_free < max_free) + rlist->max_free = max_free; + else + max_free = rlist->max_free; rtree = rlist; rlist = y; } @@ -1059,11 +1076,12 @@ * Returns: the new root. */ static vm_map_entry_t -vm_map_entry_splay(vm_offset_t addr, vm_map_entry_t root) +vm_map_entry_splay(vm_offset_t addr, vm_map_t map) { - vm_map_entry_t llist, rlist; + vm_map_entry_t llist, rlist, root; - root = vm_map_splay_split(addr, 0, root, &llist, &rlist); + llist = rlist = NULL; + root = vm_map_splay_split(addr, 0, map->root, &llist, &rlist); if (root != NULL) { /* do nothing */ } else if (llist != NULL) { @@ -1099,21 +1117,41 @@ vm_map_entry_link(vm_map_t map, vm_map_entry_t entry) { - vm_map_entry_t llist, rlist, root; + vm_map_entry_t llist, rlist, root, y; CTR3(KTR_VM, "vm_map_entry_link: map %p, nentries %d, entry %p", map, map->nentries, entry); VM_MAP_ASSERT_LOCKED(map); map->nentries++; - root = map->root; - root = vm_map_splay_split(entry->start, 0, root, &llist, &rlist); - KASSERT(root == NULL, - ("vm_map_entry_link: link object already mapped")); + llist = rlist = NULL; + root = vm_map_splay_split(entry->start, 0, map->root, &llist, &rlist); + if (root == NULL) + entry->left = entry->right = NULL; + else if (entry->start == root->start) { + KASSERT(entry->end < root->end, + ("vm_map_entry_link: new entry overflows")); + root->offset += entry->end - root->start; + root->start = entry->end; + entry->right = NULL; + entry->left = root->left; + root->left = NULL; + SPLAY_LEFT_STEP(root, y, rlist, false); + } else if (root->end == entry->end) { + KASSERT(entry->start > root->start, + ("vm_map_entry_link: new entry overflows")); + root->end = entry->start; + entry->left = NULL; + entry->right = root->right; + root->right = NULL; + SPLAY_RIGHT_STEP(root, y, llist, false); + } else + KASSERT(true, + ("vm_map_entry_link: mismatched endpoints")); entry->prev = (llist == NULL) ? &map->header : llist; entry->next = (rlist == NULL) ? &map->header : rlist; entry->prev->next = entry->next->prev = entry; - root = vm_map_splay_merge(entry, llist, rlist, NULL, NULL); + root = vm_map_splay_merge(entry, llist, rlist, entry->left, entry->right); map->root = entry; VM_MAP_ASSERT_CONSISTENT(map); } @@ -1132,12 +1170,8 @@ vm_map_entry_t llist, rlist, root, y; VM_MAP_ASSERT_LOCKED(map); - llist = entry->prev; - rlist = entry->next; - llist->next = rlist; - rlist->prev = llist; - root = map->root; - root = vm_map_splay_split(entry->start, 0, root, &llist, &rlist); + llist = rlist = NULL; + root = vm_map_splay_split(entry->start, 0, map->root, &llist, &rlist); KASSERT(root != NULL, ("vm_map_entry_unlink: unlink object not mapped")); @@ -1174,6 +1208,9 @@ root = NULL; break; } + y = entry->next; + y->prev = entry->prev; + y->prev->next = y; if (root != NULL) root = vm_map_splay_merge(root, llist, rlist, root->left, root->right); @@ -1200,8 +1237,8 @@ vm_map_entry_t llist, rlist, root; VM_MAP_ASSERT_LOCKED(map); - root = map->root; - root = vm_map_splay_split(entry->start, 0, root, &llist, &rlist); + llist = rlist = NULL; + root = vm_map_splay_split(entry->start, 0, map->root, &llist, &rlist); KASSERT(root != NULL, ("vm_map_entry_resize_free: resize_free object not mapped")); vm_map_splay_findnext(root, &rlist); @@ -1253,7 +1290,8 @@ * change the map. Thus, the map's timestamp need not change * on a temporary upgrade. */ - map->root = cur = vm_map_entry_splay(address, cur); + VM_MAP_ASSERT_CONSISTENT(map); + map->root = cur = vm_map_entry_splay(address, map); VM_MAP_ASSERT_CONSISTENT(map); if (!locked) sx_downgrade(&map->lock); @@ -1538,8 +1576,8 @@ * After splay, if start comes before root node, then there * must be a gap from start to the root. */ - root = vm_map_splay_split(start, length, map->root, - &llist, &rlist); + llist = rlist = NULL; + root = vm_map_splay_split(start, length, map->root, &llist, &rlist); if (root != NULL) start = root->end; else if (rlist != NULL) { @@ -1573,8 +1611,7 @@ /* * Splay for the least large-enough gap in the right subtree. */ - llist = NULL; - rlist = NULL; + llist = rlist = NULL; for (left_length = 0; ; left_length = root->left != NULL ? root->left->max_free : root->start - llist->end) { @@ -2068,8 +2105,6 @@ *new_entry = *entry; new_entry->end = start; - entry->offset += (start - entry->start); - entry->start = start; if (new_entry->cred != NULL) crhold(entry->cred); @@ -2150,7 +2185,7 @@ new_entry = vm_map_entry_create(map); *new_entry = *entry; - new_entry->start = entry->end = end; + new_entry->start = end; new_entry->offset += (end - entry->start); if (new_entry->cred != NULL) crhold(entry->cred);