Index: vm_map.c
===================================================================
--- vm_map.c
+++ vm_map.c
@@ -2059,16 +2059,74 @@
}
/*
+ * vm_map_entry_attach_backing_object:
+ *
+ * Allocate an object to back a map entry.
+ */
+static inline void
+vm_map_entry_attach_backing_object(vm_map_entry_t entry)
+{
+ vm_object_t object;
+
+ KASSERT(entry->object.vm_object == NULL,
+ ("map entry %p has backing object", entry));
+ KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0,
+ ("map entry %p is a submap", entry));
+ object = vm_object_allocate(OBJT_DEFAULT,
+ atop(entry->end - entry->start));
+ entry->object.vm_object = object;
+ entry->offset = 0;
+ if (entry->cred != NULL) {
+ object->cred = entry->cred;
+ object->charge = entry->end - entry->start;
+ entry->cred = NULL;
+ }
+}
+
+/*
+ * vm_map_entry_find_backing_object:
+ *
+ * If there is no object backing this entry, we might as well create one
+ * now. If we defer it, an object can get created after the map is
+ * clipped, and individual objects will be created for the split-up map.
+ * This is a bit of a hack, but is also about the best place to put this
+ * improvement.
+ */
+static inline void
+vm_map_entry_find_backing_object(vm_map_t map, vm_map_entry_t entry)
+{
+
+ VM_MAP_ASSERT_LOCKED(map);
+ KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0,
+ ("map entry %p is a submap", entry));
+ if (entry->object.vm_object == NULL && !map->system_map &&
+ (entry->eflags & MAP_ENTRY_GUARD) == 0)
+ vm_map_entry_attach_backing_object(entry);
+ else if (entry->object.vm_object != NULL &&
+ ((entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) &&
+ entry->cred != NULL) {
+ VM_OBJECT_WLOCK(entry->object.vm_object);
+ KASSERT(entry->object.vm_object->cred == NULL,
+ ("OVERCOMMIT: %s: both cred e %p", __func__, entry));
+ entry->object.vm_object->cred = entry->cred;
+ entry->object.vm_object->charge = entry->end - entry->start;
+ VM_OBJECT_WUNLOCK(entry->object.vm_object);
+ entry->cred = NULL;
+ }
+}
+
+/*
* vm_map_clip_start: [ internal use only ]
*
* Asserts that the given entry begins at or after
* the specified address; if necessary,
* it splits the entry into two.
*/
-#define vm_map_clip_start(map, entry, startaddr) \
-{ \
- if (startaddr > entry->start) \
- _vm_map_clip_start(map, entry, startaddr); \
+static inline void
+vm_map_clip_start(vm_map_t map, vm_map_entry_t entry, vm_offset_t start)
+{
+ if (start > entry->start)
+ _vm_map_clip_start(map, entry, start);
}
/*
@@ -2090,38 +2148,7 @@
* starting address.
*/
vm_map_simplify_entry(map, entry);
-
- /*
- * If there is no object backing this entry, we might as well create
- * one now. If we defer it, an object can get created after the map
- * is clipped, and individual objects will be created for the split-up
- * map. This is a bit of a hack, but is also about the best place to
- * put this improvement.
- */
- if (entry->object.vm_object == NULL && !map->system_map &&
- (entry->eflags & MAP_ENTRY_GUARD) == 0) {
- vm_object_t object;
- object = vm_object_allocate(OBJT_DEFAULT,
- atop(entry->end - entry->start));
- entry->object.vm_object = object;
- entry->offset = 0;
- if (entry->cred != NULL) {
- object->cred = entry->cred;
- object->charge = entry->end - entry->start;
- entry->cred = NULL;
- }
- } else if (entry->object.vm_object != NULL &&
- ((entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) &&
- entry->cred != NULL) {
- VM_OBJECT_WLOCK(entry->object.vm_object);
- KASSERT(entry->object.vm_object->cred == NULL,
- ("OVERCOMMIT: vm_entry_clip_start: both cred e %p", entry));
- entry->object.vm_object->cred = entry->cred;
- entry->object.vm_object->charge = entry->end - entry->start;
- VM_OBJECT_WUNLOCK(entry->object.vm_object);
- entry->cred = NULL;
- }
-
+ vm_map_entry_find_backing_object(map, entry);
new_entry = vm_map_entry_create(map);
*new_entry = *entry;
@@ -2147,16 +2174,33 @@
}
/*
+ *
+ */
+static inline vm_map_entry_t
+vm_map_lookup_clip_start(vm_map_t map, vm_offset_t start, bool modify,
+ bool modify)
+{
+ vm_map_entry_t entry;
+
+ if (!vm_map_lookup_entry(map, start, &entry))
+ entry = entry->next;
+ else if (modify)
+ vm_map_clip_start(map, entry, start);
+ return (entry);
+}
+
+/*
* vm_map_clip_end: [ internal use only ]
*
* Asserts that the given entry ends at or before
* the specified address; if necessary,
* it splits the entry into two.
*/
-#define vm_map_clip_end(map, entry, endaddr) \
-{ \
- if ((endaddr) < (entry->end)) \
- _vm_map_clip_end((map), (entry), (endaddr)); \
+static inline void
+vm_map_clip_end(vm_map_t map, vm_map_entry_t entry, vm_offset_t end)
+{
+ if (end < entry->end)
+ _vm_map_clip_end(map, entry, end);
}
/*
@@ -2173,39 +2217,9 @@
("_vm_map_clip_end: invalid clip of entry %p", entry));
/*
- * If there is no object backing this entry, we might as well create
- * one now. If we defer it, an object can get created after the map
- * is clipped, and individual objects will be created for the split-up
- * map. This is a bit of a hack, but is also about the best place to
- * put this improvement.
- */
- if (entry->object.vm_object == NULL && !map->system_map &&
- (entry->eflags & MAP_ENTRY_GUARD) == 0) {
- vm_object_t object;
- object = vm_object_allocate(OBJT_DEFAULT,
- atop(entry->end - entry->start));
- entry->object.vm_object = object;
- entry->offset = 0;
- if (entry->cred != NULL) {
- object->cred = entry->cred;
- object->charge = entry->end - entry->start;
- entry->cred = NULL;
- }
- } else if (entry->object.vm_object != NULL &&
- ((entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) &&
- entry->cred != NULL) {
- VM_OBJECT_WLOCK(entry->object.vm_object);
- KASSERT(entry->object.vm_object->cred == NULL,
- ("OVERCOMMIT: vm_entry_clip_end: both cred e %p", entry));
- entry->object.vm_object->cred = entry->cred;
- entry->object.vm_object->charge = entry->end - entry->start;
- VM_OBJECT_WUNLOCK(entry->object.vm_object);
- entry->cred = NULL;
- }
-
- /*
* Create a new entry and insert it AFTER the specified entry
*/
+ vm_map_entry_find_backing_object(map, entry);
new_entry = vm_map_entry_create(map);
*new_entry = *entry;
@@ -2260,11 +2274,7 @@
VM_MAP_RANGE_CHECK(map, start, end);
- if (vm_map_lookup_entry(map, start, &entry)) {
- vm_map_clip_start(map, entry, start);
- } else
- entry = entry->next;
-
+ entry = vm_map_lookup_clip_start(map, start, true);
vm_map_clip_end(map, entry, end);
if ((entry->start == start) && (entry->end == end) &&
@@ -2418,11 +2428,7 @@
VM_MAP_RANGE_CHECK(map, start, end);
- if (vm_map_lookup_entry(map, start, &entry)) {
- vm_map_clip_start(map, entry, start);
- } else {
- entry = entry->next;
- }
+ entry = vm_map_lookup_clip_start(map, start, true);
/*
* Make a first pass to check for protection violations.
@@ -2611,12 +2617,7 @@
*/
VM_MAP_RANGE_CHECK(map, start, end);
- if (vm_map_lookup_entry(map, start, &entry)) {
- if (modify_map)
- vm_map_clip_start(map, entry, start);
- } else {
- entry = entry->next;
- }
+ entry = vm_map_lookup_clip_start(map, start, modify_map);
if (modify_map) {
/*
@@ -2747,7 +2748,6 @@
vm_inherit_t new_inheritance)
{
vm_map_entry_t entry;
- vm_map_entry_t temp_entry;
switch (new_inheritance) {
case VM_INHERIT_NONE:
@@ -2762,11 +2762,7 @@
return (KERN_SUCCESS);
vm_map_lock(map);
VM_MAP_RANGE_CHECK(map, start, end);
- if (vm_map_lookup_entry(map, start, &temp_entry)) {
- entry = temp_entry;
- vm_map_clip_start(map, entry, start);
- } else
- entry = temp_entry->next;
+ entry = vm_map_lookup_clip_start(map, start, true);
while (entry->start < end) {
vm_map_clip_end(map, entry, end);
if ((entry->eflags & MAP_ENTRY_GUARD) == 0 ||
@@ -2856,7 +2852,7 @@
last_timestamp = map->timestamp;
continue;
}
- vm_map_clip_start(map, entry, start);
+ entry = vm_map_lookup_clip_start(map, start, true);
vm_map_clip_end(map, entry, end);
/*
* Mark the entry in case the map lock is released. (See
@@ -3092,7 +3088,7 @@
last_timestamp = map->timestamp;
continue;
}
- vm_map_clip_start(map, entry, start);
+ entry = vm_map_lookup_clip_start(map, start, true);
vm_map_clip_end(map, entry, end);
/*
* Mark the entry in case the map lock is released. (See
@@ -3499,7 +3495,6 @@
vm_map_delete(vm_map_t map, vm_offset_t start, vm_offset_t end)
{
vm_map_entry_t entry;
- vm_map_entry_t first_entry;
VM_MAP_ASSERT_LOCKED(map);
if (start == end)
@@ -3508,12 +3503,7 @@
/*
* Find the start of the region, and clip it
*/
- if (!vm_map_lookup_entry(map, start, &first_entry))
- entry = first_entry->next;
- else {
- entry = first_entry;
- vm_map_clip_start(map, entry, start);
- }
+ entry = vm_map_lookup_clip_start(map, start, true);
/*
* Step through all entries in this region
@@ -3531,7 +3521,6 @@
vm_map_entry_system_wired_count(entry) != 0)) {
unsigned int last_timestamp;
vm_offset_t saved_start;
- vm_map_entry_t tmp_entry;
saved_start = entry->start;
entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP;
@@ -3545,14 +3534,8 @@
* Specifically, the entry may have been
* clipped, merged, or deleted.
*/
- if (!vm_map_lookup_entry(map, saved_start,
- &tmp_entry))
- entry = tmp_entry->next;
- else {
- entry = tmp_entry;
- vm_map_clip_start(map, entry,
- saved_start);
- }
+ entry = vm_map_lookup_clip_start(map,
+ saved_start, true);
}
continue;
}
@@ -3885,16 +3868,8 @@
*/
object = old_entry->object.vm_object;
if (object == NULL) {
- object = vm_object_allocate(OBJT_DEFAULT,
- atop(old_entry->end - old_entry->start));
- old_entry->object.vm_object = object;
- old_entry->offset = 0;
- if (old_entry->cred != NULL) {
- object->cred = old_entry->cred;
- object->charge = old_entry->end -
- old_entry->start;
- old_entry->cred = NULL;
- }
+ vm_map_entry_attach_backing_object(old_entry);
+ object = old_entry->object.vm_object;
}
/*