diff --git a/share/man/man9/vm_map.9 b/share/man/man9/vm_map.9 --- a/share/man/man9/vm_map.9 +++ b/share/man/man9/vm_map.9 @@ -49,15 +49,15 @@ function. .Bd -literal -offset indent struct vm_map { - struct vm_map_entry header; - struct sx lock; - struct mtx system_mtx; + struct vm_map_entry header; + union { + struct sx lock; + struct mtx system_mtx; + }; int nentries; vm_size_t size; u_int timestamp; - u_char needs_wakeup; - u_char system_map; - vm_flags_t flags; + u_int flags; vm_map_entry_t root; pmap_t pmap; int busy; @@ -83,12 +83,6 @@ Specifies the size of the virtual address space. .It Va timestamp Used to determine if the map has changed since its last access. -.It Va needs_wakeup -Indicates if a thread is waiting for an allocation within the map. -Used only by system maps. -.It Va system_map -Set to TRUE to indicate that map is a system map; otherwise, it belongs -to a user process. .It Va flags Map flags, described below. .It Va root @@ -106,6 +100,12 @@ Wire all future pages in this map. .It Dv MAP_BUSY_WAKEUP There are waiters for the map busy status. +.It Va MAP_NEEDS_WAKEUP +Indicates if a thread is waiting for an allocation within the map. +Used only by system maps. +.It Va MAP_SYSTEM_MAP +If set, ndicates that map is a system map; otherwise, it belongs +to a user process. .El .Pp The following flags can be passed to diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c --- a/sys/vm/vm_fault.c +++ b/sys/vm/vm_fault.c @@ -1355,7 +1355,7 @@ MPASS(status == FAULT_CONTINUE || status == FAULT_RESTART); if (status == FAULT_RESTART) return (status); - KASSERT(fs->vp == NULL || !fs->map->system_map, + KASSERT(fs->vp == NULL || !vm_map_is_system(fs->map), ("vm_fault: vnode-backed object mapped by system map")); /* diff --git a/sys/vm/vm_kern.c b/sys/vm/vm_kern.c --- a/sys/vm/vm_kern.c +++ b/sys/vm/vm_kern.c @@ -724,7 +724,7 @@ swap_release(size); return (0); } - map->needs_wakeup = TRUE; + vm_map_modflags(map, MAP_NEEDS_WAKEUP, 0); vm_map_unlock_and_wait(map, 0); } vm_map_insert(map, NULL, 0, addr, addr + size, VM_PROT_RW, VM_PROT_RW, @@ -745,8 +745,8 @@ vm_map_lock(map); (void) vm_map_delete(map, trunc_page(addr), round_page(addr + size)); - if (map->needs_wakeup) { - map->needs_wakeup = FALSE; + if ((map->flags & MAP_NEEDS_WAKEUP) != 0) { + vm_map_modflags(map, 0, MAP_NEEDS_WAKEUP); vm_map_wakeup(map); } vm_map_unlock(map); @@ -830,8 +830,7 @@ vm_size_t quantum; int domain; - vm_map_init(kernel_map, kernel_pmap, VM_MIN_KERNEL_ADDRESS, end); - kernel_map->system_map = 1; + vm_map_init_system(kernel_map, kernel_pmap, VM_MIN_KERNEL_ADDRESS, end); vm_map_lock(kernel_map); /* N.B.: cannot use kgdb to debug, starting with this assignment ... */ (void)vm_map_insert(kernel_map, NULL, 0, diff --git a/sys/vm/vm_map.h b/sys/vm/vm_map.h --- a/sys/vm/vm_map.h +++ b/sys/vm/vm_map.h @@ -75,7 +75,6 @@ * vm_map_entry_t an entry in an address map. */ -typedef u_char vm_flags_t; typedef u_int vm_eflags_t; /* @@ -203,14 +202,14 @@ */ struct vm_map { struct vm_map_entry header; /* List of entries */ - struct sx lock; /* Lock for map data */ - struct mtx system_mtx; + union { + struct sx lock; /* Lock for map data */ + struct mtx system_mtx; + }; int nentries; /* Number of entries */ vm_size_t size; /* virtual size */ u_int timestamp; /* Version number */ - u_char needs_wakeup; - u_char system_map; /* (c) Am I a system map? */ - vm_flags_t flags; /* flags for this vm_map */ + u_int flags; /* flags for this vm_map */ vm_map_entry_t root; /* Root of a binary search tree */ pmap_t pmap; /* (c) Physical map */ vm_offset_t anon_loc; @@ -221,16 +220,21 @@ }; /* - * vm_flags_t values + * vm_map flags values */ -#define MAP_WIREFUTURE 0x01 /* wire all future pages */ -#define MAP_BUSY_WAKEUP 0x02 /* thread(s) waiting on busy state */ -#define MAP_IS_SUB_MAP 0x04 /* has parent */ -#define MAP_ASLR 0x08 /* enabled ASLR */ -#define MAP_ASLR_IGNSTART 0x10 /* ASLR ignores data segment */ -#define MAP_REPLENISH 0x20 /* kmapent zone needs to be refilled */ -#define MAP_WXORX 0x40 /* enforce W^X */ -#define MAP_ASLR_STACK 0x80 /* stack location is randomized */ +#define MAP_WIREFUTURE 0x00000001 /* wire all future pages */ +#define MAP_BUSY_WAKEUP 0x00000002 /* thread(s) waiting on busy + state */ +#define MAP_IS_SUB_MAP 0x00000004 /* has parent */ +#define MAP_ASLR 0x00000008 /* enabled ASLR */ +#define MAP_ASLR_IGNSTART 0x00000010 /* ASLR ignores data segment */ +#define MAP_REPLENISH 0x00000020 /* kmapent zone needs to be + refilled */ +#define MAP_WXORX 0x00000040 /* enforce W^X */ +#define MAP_ASLR_STACK 0x00000080 /* stack location is + randomized */ +#define MAP_NEEDS_WAKEUP 0x40000000 +#define MAP_SYSTEM_MAP 0x80000000 #ifdef _KERNEL #if defined(KLD_MODULE) && !defined(KLD_TIED) @@ -261,7 +265,7 @@ } static __inline void -vm_map_modflags(vm_map_t map, vm_flags_t set, vm_flags_t clear) +vm_map_modflags(vm_map_t map, u_int set, u_int clear) { map->flags = (map->flags | set) & ~clear; } @@ -276,6 +280,12 @@ return (true); } +static inline bool +vm_map_is_system(vm_map_t map) +{ + return ((map->flags & MAP_SYSTEM_MAP) != 0); +} + #endif /* KLD_MODULE */ #endif /* _KERNEL */ @@ -484,6 +494,7 @@ vm_offset_t vm_map_findspace(vm_map_t, vm_offset_t, vm_size_t); int vm_map_inherit (vm_map_t, vm_offset_t, vm_offset_t, vm_inherit_t); void vm_map_init(vm_map_t, pmap_t, vm_offset_t, vm_offset_t); +void vm_map_init_system(vm_map_t, pmap_t, vm_offset_t, vm_offset_t); int vm_map_insert (vm_map_t, vm_object_t, vm_ooffset_t, vm_offset_t, vm_offset_t, vm_prot_t, vm_prot_t, int); int vm_map_lookup (vm_map_t *, vm_offset_t, vm_prot_t, vm_map_entry_t *, vm_object_t *, vm_pindex_t *, vm_prot_t *, boolean_t *); diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c --- a/sys/vm/vm_map.c +++ b/sys/vm/vm_map.c @@ -292,9 +292,7 @@ vm = (struct vmspace *)mem; map = &vm->vm_map; - memset(map, 0, sizeof(*map)); - mtx_init(&map->system_mtx, "vm map (system)", NULL, - MTX_DEF | MTX_DUPOK); + memset(map, 0, sizeof(*map)); /* set MAP_SYSTEM_MAP to false */ sx_init(&map->lock, "vm map (user)"); PMAP_LOCK_INIT(vmspace_pmap(vm)); return (0); @@ -525,7 +523,7 @@ _vm_map_lock(vm_map_t map, const char *file, int line) { - if (map->system_map) + if (vm_map_is_system(map)) mtx_lock_flags_(&map->system_mtx, 0, file, line); else sx_xlock_(&map->lock, file, line); @@ -616,7 +614,7 @@ _vm_map_assert_locked(vm_map_t map, const char *file, int line) { - if (map->system_map) + if (vm_map_is_system(map)) mtx_assert_(&map->system_mtx, MA_OWNED, file, line); else sx_assert_(&map->lock, SA_XLOCKED, file, line); @@ -659,7 +657,7 @@ { VM_MAP_UNLOCK_CONSISTENT(map); - if (map->system_map) { + if (vm_map_is_system(map)) { #ifndef UMA_USE_DMAP if (map == kernel_map && (map->flags & MAP_REPLENISH) != 0) { uma_prealloc(kmapentzone, 1); @@ -677,7 +675,7 @@ _vm_map_lock_read(vm_map_t map, const char *file, int line) { - if (map->system_map) + if (vm_map_is_system(map)) mtx_lock_flags_(&map->system_mtx, 0, file, line); else sx_slock_(&map->lock, file, line); @@ -687,7 +685,7 @@ _vm_map_unlock_read(vm_map_t map, const char *file, int line) { - if (map->system_map) { + if (vm_map_is_system(map)) { KASSERT((map->flags & MAP_REPLENISH) == 0, ("%s: MAP_REPLENISH leaked", __func__)); mtx_unlock_flags_(&map->system_mtx, 0, file, line); @@ -702,7 +700,7 @@ { int error; - error = map->system_map ? + error = vm_map_is_system(map) ? !mtx_trylock_flags_(&map->system_mtx, 0, file, line) : !sx_try_xlock_(&map->lock, file, line); if (error == 0) @@ -715,7 +713,7 @@ { int error; - error = map->system_map ? + error = vm_map_is_system(map) ? !mtx_trylock_flags_(&map->system_mtx, 0, file, line) : !sx_try_slock_(&map->lock, file, line); return (error == 0); @@ -736,7 +734,7 @@ { unsigned int last_timestamp; - if (map->system_map) { + if (vm_map_is_system(map)) { mtx_assert_(&map->system_mtx, MA_OWNED, file, line); } else { if (!sx_try_upgrade_(&map->lock, file, line)) { @@ -762,7 +760,7 @@ _vm_map_lock_downgrade(vm_map_t map, const char *file, int line) { - if (map->system_map) { + if (vm_map_is_system(map)) { KASSERT((map->flags & MAP_REPLENISH) == 0, ("%s: MAP_REPLENISH leaked", __func__)); mtx_assert_(&map->system_mtx, MA_OWNED, file, line); @@ -782,10 +780,9 @@ vm_map_locked(vm_map_t map) { - if (map->system_map) + if (vm_map_is_system(map)) return (mtx_owned(&map->system_mtx)); - else - return (sx_xlocked(&map->lock)); + return (sx_xlocked(&map->lock)); } /* @@ -808,7 +805,7 @@ VM_MAP_UNLOCK_CONSISTENT(map); mtx_lock(&map_sleep_mtx); - if (map->system_map) { + if (vm_map_is_system(map)) { KASSERT((map->flags & MAP_REPLENISH) == 0, ("%s: MAP_REPLENISH leaked", __func__)); mtx_unlock_flags_(&map->system_mtx, 0, file, line); @@ -866,7 +863,7 @@ VM_MAP_ASSERT_LOCKED(map); while (map->busy) { vm_map_modflags(map, MAP_BUSY_WAKEUP, 0); - if (map->system_map) + if (vm_map_is_system(map)) msleep(&map->busy, &map->system_mtx, 0, "mbusy", 0); else sx_sleep(&map->busy, &map->lock, 0, "mbusy", 0); @@ -889,8 +886,6 @@ { map->header.eflags = MAP_ENTRY_HEADER; - map->needs_wakeup = FALSE; - map->system_map = 0; map->pmap = pmap; map->header.end = min; map->header.start = max; @@ -908,13 +903,19 @@ void vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max) { - _vm_map_init(map, pmap, min, max); - mtx_init(&map->system_mtx, "vm map (system)", NULL, - MTX_DEF | MTX_DUPOK); sx_init(&map->lock, "vm map (user)"); } +void +vm_map_init_system(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max) +{ + _vm_map_init(map, pmap, min, max); + vm_map_modflags(map, MAP_SYSTEM_MAP, 0); + mtx_init(&map->system_mtx, "vm map (system)", NULL, MTX_DEF | + MTX_DUPOK); +} + /* * vm_map_entry_dispose: [ internal use only ] * @@ -923,7 +924,7 @@ static void vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry) { - uma_zfree(map->system_map ? kmapentzone : mapentzone, entry); + uma_zfree(vm_map_is_system(map) ? kmapentzone : mapentzone, entry); } /* @@ -957,7 +958,7 @@ } } else #endif - if (map->system_map) { + if (vm_map_is_system(map)) { new_entry = uma_zalloc(kmapentzone, M_NOWAIT); } else { new_entry = uma_zalloc(mapentzone, M_WAITOK); @@ -2429,7 +2430,7 @@ VM_MAP_ASSERT_LOCKED(map); KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0, ("map entry %p is a submap", entry)); - if (entry->object.vm_object == NULL && !map->system_map && + if (entry->object.vm_object == NULL && !vm_map_is_system(map) && (entry->eflags & MAP_ENTRY_GUARD) == 0) vm_map_entry_back(entry); else if (entry->object.vm_object != NULL && @@ -2494,7 +2495,7 @@ vm_map_entry_t new_entry; int bdry_idx; - if (!map->system_map) + if (!vm_map_is_system(map)) WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "%s: map %p entry %p start 0x%jx", __func__, map, entry, (uintmax_t)startaddr); @@ -2537,7 +2538,7 @@ vm_map_entry_t entry; int rv; - if (!map->system_map) + if (!vm_map_is_system(map)) WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "%s: map %p start 0x%jx prev %p", __func__, map, (uintmax_t)start, prev_entry); @@ -2567,7 +2568,7 @@ vm_map_entry_t new_entry; int bdry_idx; - if (!map->system_map) + if (!vm_map_is_system(map)) WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "%s: map %p entry %p end 0x%jx", __func__, map, entry, (uintmax_t)endaddr); @@ -3937,7 +3938,7 @@ MPASS(entry->cred == NULL); MPASS((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0); MPASS(object == NULL); - vm_map_entry_deallocate(entry, map->system_map); + vm_map_entry_deallocate(entry, vm_map_is_system(map)); return; } @@ -3988,7 +3989,7 @@ } VM_OBJECT_WUNLOCK(object); } - if (map->system_map) + if (vm_map_is_system(map)) vm_map_entry_deallocate(entry, TRUE); else { entry->defer_next = curthread->td_map_def_user; @@ -4711,7 +4712,7 @@ p->p_textvp == NULL)) return (KERN_FAILURE); - MPASS(!map->system_map); + MPASS(!vm_map_is_system(map)); lmemlim = lim_cur(curthread, RLIMIT_MEMLOCK); stacklim = lim_cur(curthread, RLIMIT_STACK); @@ -5119,7 +5120,7 @@ /* * Create an object if necessary. */ - if (entry->object.vm_object == NULL && !map->system_map) { + if (entry->object.vm_object == NULL && !vm_map_is_system(map)) { if (vm_map_lock_upgrade(map)) goto RetryLookup; entry->object.vm_object = vm_object_allocate_anon(atop(size), @@ -5207,7 +5208,7 @@ /* * Fail if an object should be created. */ - if (entry->object.vm_object == NULL && !map->system_map) + if (entry->object.vm_object == NULL && !vm_map_is_system(map)) return (KERN_FAILURE); /* diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -1844,7 +1844,7 @@ long res; map = &vmspace->vm_map; - KASSERT(!map->system_map, ("system map")); + KASSERT(!vm_map_is_system(map), ("system map")); sx_assert(&map->lock, SA_LOCKED); res = 0; VM_MAP_ENTRY_FOREACH(entry, map) {