diff --git a/sys/vm/vm.h b/sys/vm/vm.h index 499a504e3328..5fac6b3f2e78 100644 --- a/sys/vm/vm.h +++ b/sys/vm/vm.h @@ -1,177 +1,183 @@ /*- * SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU) * * Copyright (c) 1991, 1993 * The Regents of the University of California. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * Copyright (c) 1987, 1990 Carnegie-Mellon University. * All rights reserved. * * Authors: Avadis Tevanian, Jr., Michael Wayne Young * * Permission to use, copy, modify and distribute this software and * its documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. * * Carnegie Mellon requests users of this software to return to * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 * * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. */ #ifndef VM_H #define VM_H #include typedef char vm_inherit_t; /* inheritance codes */ #define VM_INHERIT_SHARE ((vm_inherit_t) 0) #define VM_INHERIT_COPY ((vm_inherit_t) 1) #define VM_INHERIT_NONE ((vm_inherit_t) 2) #define VM_INHERIT_ZERO ((vm_inherit_t) 3) #define VM_INHERIT_DEFAULT VM_INHERIT_COPY typedef u_char vm_prot_t; /* protection codes */ #define VM_PROT_NONE ((vm_prot_t) 0x00) #define VM_PROT_READ ((vm_prot_t) 0x01) #define VM_PROT_WRITE ((vm_prot_t) 0x02) #define VM_PROT_EXECUTE ((vm_prot_t) 0x04) #define VM_PROT_COPY ((vm_prot_t) 0x08) /* copy-on-read */ #define VM_PROT_PRIV_FLAG ((vm_prot_t) 0x10) #define VM_PROT_FAULT_LOOKUP VM_PROT_PRIV_FLAG #define VM_PROT_QUICK_NOFAULT VM_PROT_PRIV_FLAG /* same to save bits */ #define VM_PROT_ALL (VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE) #define VM_PROT_RW (VM_PROT_READ|VM_PROT_WRITE) #define VM_PROT_DEFAULT VM_PROT_ALL enum obj_type { OBJT_RESERVED = 0, /* was OBJT_DEFAULT */ OBJT_SWAP, OBJT_DEFAULT = OBJT_SWAP, OBJT_VNODE, OBJT_DEVICE, OBJT_PHYS, OBJT_DEAD, OBJT_SG, OBJT_MGTDEVICE, OBJT_FIRST_DYN, }; typedef u_char objtype_t; union vm_map_object; typedef union vm_map_object vm_map_object_t; struct vm_map_entry; typedef struct vm_map_entry *vm_map_entry_t; struct vm_map; typedef struct vm_map *vm_map_t; struct vm_object; typedef struct vm_object *vm_object_t; #ifndef _KERNEL /* * This is defined in for the kernel so that non-vm kernel * sources (mainly Mach-derived ones such as ddb) don't have to include * vm stuff. Defining it there for applications might break things. * Define it here for "applications" that include vm headers (e.g., * genassym). */ #ifndef HAVE_BOOLEAN typedef int boolean_t; #endif /* * The exact set of memory attributes is machine dependent. However, * every machine is required to define VM_MEMATTR_DEFAULT and * VM_MEMATTR_UNCACHEABLE. */ typedef char vm_memattr_t; /* memory attribute codes */ /* * This is defined in for the kernel so that vnode_if.h * doesn't have to include . */ struct vm_page; typedef struct vm_page *vm_page_t; #endif /* _KERNEL */ struct vm_reserv; typedef struct vm_reserv *vm_reserv_t; /* * Information passed from the machine-independent VM initialization code * for use by machine-dependant code (mainly for MMU support) */ struct kva_md_info { vm_offset_t buffer_sva; vm_offset_t buffer_eva; vm_offset_t clean_sva; vm_offset_t clean_eva; }; /* bits from overcommit */ #define SWAP_RESERVE_FORCE_ON (1 << 0) #define SWAP_RESERVE_RLIMIT_ON (1 << 1) #define SWAP_RESERVE_ALLOW_NONWIRED (1 << 2) +#ifdef NUMA +#define __numa_used +#else +#define __numa_used __unused +#endif + #ifdef _KERNEL struct ucred; void vm_ksubmap_init(struct kva_md_info *); bool swap_reserve(vm_ooffset_t incr); bool swap_reserve_by_cred(vm_ooffset_t incr, struct ucred *cred); void swap_reserve_force(vm_ooffset_t incr); void swap_release(vm_ooffset_t decr); void swap_release_by_cred(vm_ooffset_t decr, struct ucred *cred); void swapper(void); extern struct kva_md_info kmi; #define VA_IS_CLEANMAP(va) \ ((va) >= kmi.clean_sva && (va) < kmi.clean_eva) extern int old_mlock; extern int vm_ndomains; extern int vm_overcommit; #endif /* _KERNEL */ #endif /* VM_H */ diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h index 37cb1374b9f6..221f58b8fb04 100644 --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -1,1024 +1,1024 @@ /*- * SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU) * * Copyright (c) 1991, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * The Mach Operating System project at Carnegie-Mellon University. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * * Copyright (c) 1987, 1990 Carnegie-Mellon University. * All rights reserved. * * Authors: Avadis Tevanian, Jr., Michael Wayne Young * * Permission to use, copy, modify and distribute this software and * its documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. * * Carnegie Mellon requests users of this software to return to * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 * * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. */ /* * Resident memory system definitions. */ #ifndef _VM_PAGE_ #define _VM_PAGE_ #include #include /* * Management of resident (logical) pages. * * A small structure is kept for each resident * page, indexed by page number. Each structure * is an element of several collections: * * A radix tree used to quickly * perform object/offset lookups * * A list of all pages for a given object, * so they can be quickly deactivated at * time of deallocation. * * An ordered list of pages due for pageout. * * In addition, the structure contains the object * and offset to which this page belongs (for pageout), * and sundry status bits. * * In general, operations on this structure's mutable fields are * synchronized using either one of or a combination of locks. If a * field is annotated with two of these locks then holding either is * sufficient for read access but both are required for write access. * The queue lock for a page depends on the value of its queue field and is * described in detail below. * * The following annotations are possible: * (A) the field must be accessed using atomic(9) and may require * additional synchronization. * (B) the page busy lock. * (C) the field is immutable. * (F) the per-domain lock for the free queues. * (M) Machine dependent, defined by pmap layer. * (O) the object that the page belongs to. * (Q) the page's queue lock. * * The busy lock is an embedded reader-writer lock that protects the * page's contents and identity (i.e., its tuple) as * well as certain valid/dirty modifications. To avoid bloating the * the page structure, the busy lock lacks some of the features available * the kernel's general-purpose synchronization primitives. As a result, * busy lock ordering rules are not verified, lock recursion is not * detected, and an attempt to xbusy a busy page or sbusy an xbusy page * results will trigger a panic rather than causing the thread to block. * vm_page_sleep_if_busy() can be used to sleep until the page's busy * state changes, after which the caller must re-lookup the page and * re-evaluate its state. vm_page_busy_acquire() will block until * the lock is acquired. * * The valid field is protected by the page busy lock (B) and object * lock (O). Transitions from invalid to valid are generally done * via I/O or zero filling and do not require the object lock. * These must be protected with the busy lock to prevent page-in or * creation races. Page invalidation generally happens as a result * of truncate or msync. When invalidated, pages must not be present * in pmap and must hold the object lock to prevent concurrent * speculative read-only mappings that do not require busy. I/O * routines may check for validity without a lock if they are prepared * to handle invalidation races with higher level locks (vnode) or are * unconcerned with races so long as they hold a reference to prevent * recycling. When a valid bit is set while holding a shared busy * lock (A) atomic operations are used to protect against concurrent * modification. * * In contrast, the synchronization of accesses to the page's * dirty field is a mix of machine dependent (M) and busy (B). In * the machine-independent layer, the page busy must be held to * operate on the field. However, the pmap layer is permitted to * set all bits within the field without holding that lock. If the * underlying architecture does not support atomic read-modify-write * operations on the field's type, then the machine-independent * layer uses a 32-bit atomic on the aligned 32-bit word that * contains the dirty field. In the machine-independent layer, * the implementation of read-modify-write operations on the * field is encapsulated in vm_page_clear_dirty_mask(). An * exclusive busy lock combined with pmap_remove_{write/all}() is the * only way to ensure a page can not become dirty. I/O generally * removes the page from pmap to ensure exclusive access and atomic * writes. * * The ref_count field tracks references to the page. References that * prevent the page from being reclaimable are called wirings and are * counted in the low bits of ref_count. The containing object's * reference, if one exists, is counted using the VPRC_OBJREF bit in the * ref_count field. Additionally, the VPRC_BLOCKED bit is used to * atomically check for wirings and prevent new wirings via * pmap_extract_and_hold(). When a page belongs to an object, it may be * wired only when the object is locked, or the page is busy, or by * pmap_extract_and_hold(). As a result, if the object is locked and the * page is not busy (or is exclusively busied by the current thread), and * the page is unmapped, its wire count will not increase. The ref_count * field is updated using atomic operations in most cases, except when it * is known that no other references to the page exist, such as in the page * allocator. A page may be present in the page queues, or even actively * scanned by the page daemon, without an explicitly counted referenced. * The page daemon must therefore handle the possibility of a concurrent * free of the page. * * The queue state of a page consists of the queue and act_count fields of * its atomically updated state, and the subset of atomic flags specified * by PGA_QUEUE_STATE_MASK. The queue field contains the page's page queue * index, or PQ_NONE if it does not belong to a page queue. To modify the * queue field, the page queue lock corresponding to the old value must be * held, unless that value is PQ_NONE, in which case the queue index must * be updated using an atomic RMW operation. There is one exception to * this rule: the page daemon may transition the queue field from * PQ_INACTIVE to PQ_NONE immediately prior to freeing the page during an * inactive queue scan. At that point the page is already dequeued and no * other references to that vm_page structure can exist. The PGA_ENQUEUED * flag, when set, indicates that the page structure is physically inserted * into the queue corresponding to the page's queue index, and may only be * set or cleared with the corresponding page queue lock held. * * To avoid contention on page queue locks, page queue operations (enqueue, * dequeue, requeue) are batched using fixed-size per-CPU queues. A * deferred operation is requested by setting one of the flags in * PGA_QUEUE_OP_MASK and inserting an entry into a batch queue. When a * queue is full, an attempt to insert a new entry will lock the page * queues and trigger processing of the pending entries. The * type-stability of vm_page structures is crucial to this scheme since the * processing of entries in a given batch queue may be deferred * indefinitely. In particular, a page may be freed with pending batch * queue entries. The page queue operation flags must be set using atomic * RWM operations. */ #if PAGE_SIZE == 4096 #define VM_PAGE_BITS_ALL 0xffu typedef uint8_t vm_page_bits_t; #elif PAGE_SIZE == 8192 #define VM_PAGE_BITS_ALL 0xffffu typedef uint16_t vm_page_bits_t; #elif PAGE_SIZE == 16384 #define VM_PAGE_BITS_ALL 0xffffffffu typedef uint32_t vm_page_bits_t; #elif PAGE_SIZE == 32768 #define VM_PAGE_BITS_ALL 0xfffffffffffffffflu typedef uint64_t vm_page_bits_t; #endif typedef union vm_page_astate { struct { uint16_t flags; uint8_t queue; uint8_t act_count; }; uint32_t _bits; } vm_page_astate_t; struct vm_page { union { TAILQ_ENTRY(vm_page) q; /* page queue or free list (Q) */ struct { SLIST_ENTRY(vm_page) ss; /* private slists */ } s; struct { u_long p; u_long v; } memguard; struct { void *slab; void *zone; } uma; } plinks; TAILQ_ENTRY(vm_page) listq; /* pages in same object (O) */ vm_object_t object; /* which object am I in (O) */ vm_pindex_t pindex; /* offset into object (O,P) */ vm_paddr_t phys_addr; /* physical address of page (C) */ struct md_page md; /* machine dependent stuff */ u_int ref_count; /* page references (A) */ u_int busy_lock; /* busy owners lock (A) */ union vm_page_astate a; /* state accessed atomically (A) */ uint8_t order; /* index of the buddy queue (F) */ uint8_t pool; /* vm_phys freepool index (F) */ uint8_t flags; /* page PG_* flags (P) */ uint8_t oflags; /* page VPO_* flags (O) */ int8_t psind; /* pagesizes[] index (O) */ int8_t segind; /* vm_phys segment index (C) */ /* NOTE that these must support one bit per DEV_BSIZE in a page */ /* so, on normal X86 kernels, they must be at least 8 bits wide */ vm_page_bits_t valid; /* valid DEV_BSIZE chunk map (O,B) */ vm_page_bits_t dirty; /* dirty DEV_BSIZE chunk map (M,B) */ }; /* * Special bits used in the ref_count field. * * ref_count is normally used to count wirings that prevent the page from being * reclaimed, but also supports several special types of references that do not * prevent reclamation. Accesses to the ref_count field must be atomic unless * the page is unallocated. * * VPRC_OBJREF is the reference held by the containing object. It can set or * cleared only when the corresponding object's write lock is held. * * VPRC_BLOCKED is used to atomically block wirings via pmap lookups while * attempting to tear down all mappings of a given page. The page busy lock and * object write lock must both be held in order to set or clear this bit. */ #define VPRC_BLOCKED 0x40000000u /* mappings are being removed */ #define VPRC_OBJREF 0x80000000u /* object reference, cleared with (O) */ #define VPRC_WIRE_COUNT(c) ((c) & ~(VPRC_BLOCKED | VPRC_OBJREF)) #define VPRC_WIRE_COUNT_MAX (~(VPRC_BLOCKED | VPRC_OBJREF)) /* * Page flags stored in oflags: * * Access to these page flags is synchronized by the lock on the object * containing the page (O). * * Note: VPO_UNMANAGED (used by OBJT_DEVICE, OBJT_PHYS and OBJT_SG) * indicates that the page is not under PV management but * otherwise should be treated as a normal page. Pages not * under PV management cannot be paged out via the * object/vm_page_t because there is no knowledge of their pte * mappings, and such pages are also not on any PQ queue. * */ #define VPO_KMEM_EXEC 0x01 /* kmem mapping allows execution */ #define VPO_SWAPSLEEP 0x02 /* waiting for swap to finish */ #define VPO_UNMANAGED 0x04 /* no PV management for page */ #define VPO_SWAPINPROG 0x08 /* swap I/O in progress on page */ /* * Busy page implementation details. * The algorithm is taken mostly by rwlock(9) and sx(9) locks implementation, * even if the support for owner identity is removed because of size * constraints. Checks on lock recursion are then not possible, while the * lock assertions effectiveness is someway reduced. */ #define VPB_BIT_SHARED 0x01 #define VPB_BIT_EXCLUSIVE 0x02 #define VPB_BIT_WAITERS 0x04 #define VPB_BIT_FLAGMASK \ (VPB_BIT_SHARED | VPB_BIT_EXCLUSIVE | VPB_BIT_WAITERS) #define VPB_SHARERS_SHIFT 3 #define VPB_SHARERS(x) \ (((x) & ~VPB_BIT_FLAGMASK) >> VPB_SHARERS_SHIFT) #define VPB_SHARERS_WORD(x) ((x) << VPB_SHARERS_SHIFT | VPB_BIT_SHARED) #define VPB_ONE_SHARER (1 << VPB_SHARERS_SHIFT) #define VPB_SINGLE_EXCLUSIVE VPB_BIT_EXCLUSIVE #ifdef INVARIANTS #define VPB_CURTHREAD_EXCLUSIVE \ (VPB_BIT_EXCLUSIVE | ((u_int)(uintptr_t)curthread & ~VPB_BIT_FLAGMASK)) #else #define VPB_CURTHREAD_EXCLUSIVE VPB_SINGLE_EXCLUSIVE #endif #define VPB_UNBUSIED VPB_SHARERS_WORD(0) /* Freed lock blocks both shared and exclusive. */ #define VPB_FREED (0xffffffff - VPB_BIT_SHARED) #define PQ_NONE 255 #define PQ_INACTIVE 0 #define PQ_ACTIVE 1 #define PQ_LAUNDRY 2 #define PQ_UNSWAPPABLE 3 #define PQ_COUNT 4 #ifndef VM_PAGE_HAVE_PGLIST TAILQ_HEAD(pglist, vm_page); #define VM_PAGE_HAVE_PGLIST #endif SLIST_HEAD(spglist, vm_page); #ifdef _KERNEL extern vm_page_t bogus_page; #endif /* _KERNEL */ extern struct mtx_padalign pa_lock[]; #if defined(__arm__) #define PDRSHIFT PDR_SHIFT #elif !defined(PDRSHIFT) #define PDRSHIFT 21 #endif #define pa_index(pa) ((pa) >> PDRSHIFT) #define PA_LOCKPTR(pa) ((struct mtx *)(&pa_lock[pa_index(pa) % PA_LOCK_COUNT])) #define PA_LOCKOBJPTR(pa) ((struct lock_object *)PA_LOCKPTR((pa))) #define PA_LOCK(pa) mtx_lock(PA_LOCKPTR(pa)) #define PA_TRYLOCK(pa) mtx_trylock(PA_LOCKPTR(pa)) #define PA_UNLOCK(pa) mtx_unlock(PA_LOCKPTR(pa)) #define PA_UNLOCK_COND(pa) \ do { \ if ((pa) != 0) { \ PA_UNLOCK((pa)); \ (pa) = 0; \ } \ } while (0) #define PA_LOCK_ASSERT(pa, a) mtx_assert(PA_LOCKPTR(pa), (a)) #if defined(KLD_MODULE) && !defined(KLD_TIED) #define vm_page_lock(m) vm_page_lock_KBI((m), LOCK_FILE, LOCK_LINE) #define vm_page_unlock(m) vm_page_unlock_KBI((m), LOCK_FILE, LOCK_LINE) #define vm_page_trylock(m) vm_page_trylock_KBI((m), LOCK_FILE, LOCK_LINE) #else /* !KLD_MODULE */ #define vm_page_lockptr(m) (PA_LOCKPTR(VM_PAGE_TO_PHYS((m)))) #define vm_page_lock(m) mtx_lock(vm_page_lockptr((m))) #define vm_page_unlock(m) mtx_unlock(vm_page_lockptr((m))) #define vm_page_trylock(m) mtx_trylock(vm_page_lockptr((m))) #endif #if defined(INVARIANTS) #define vm_page_assert_locked(m) \ vm_page_assert_locked_KBI((m), __FILE__, __LINE__) #define vm_page_lock_assert(m, a) \ vm_page_lock_assert_KBI((m), (a), __FILE__, __LINE__) #else #define vm_page_assert_locked(m) #define vm_page_lock_assert(m, a) #endif /* * The vm_page's aflags are updated using atomic operations. To set or clear * these flags, the functions vm_page_aflag_set() and vm_page_aflag_clear() * must be used. Neither these flags nor these functions are part of the KBI. * * PGA_REFERENCED may be cleared only if the page is locked. It is set by * both the MI and MD VM layers. However, kernel loadable modules should not * directly set this flag. They should call vm_page_reference() instead. * * PGA_WRITEABLE is set exclusively on managed pages by pmap_enter(). * When it does so, the object must be locked, or the page must be * exclusive busied. The MI VM layer must never access this flag * directly. Instead, it should call pmap_page_is_write_mapped(). * * PGA_EXECUTABLE may be set by pmap routines, and indicates that a page has * at least one executable mapping. It is not consumed by the MI VM layer. * * PGA_NOSYNC must be set and cleared with the page busy lock held. * * PGA_ENQUEUED is set and cleared when a page is inserted into or removed * from a page queue, respectively. It determines whether the plinks.q field * of the page is valid. To set or clear this flag, page's "queue" field must * be a valid queue index, and the corresponding page queue lock must be held. * * PGA_DEQUEUE is set when the page is scheduled to be dequeued from a page * queue, and cleared when the dequeue request is processed. A page may * have PGA_DEQUEUE set and PGA_ENQUEUED cleared, for instance if a dequeue * is requested after the page is scheduled to be enqueued but before it is * actually inserted into the page queue. * * PGA_REQUEUE is set when the page is scheduled to be enqueued or requeued * in its page queue. * * PGA_REQUEUE_HEAD is a special flag for enqueuing pages near the head of * the inactive queue, thus bypassing LRU. * * The PGA_DEQUEUE, PGA_REQUEUE and PGA_REQUEUE_HEAD flags must be set using an * atomic RMW operation to ensure that the "queue" field is a valid queue index, * and the corresponding page queue lock must be held when clearing any of the * flags. * * PGA_SWAP_FREE is used to defer freeing swap space to the pageout daemon * when the context that dirties the page does not have the object write lock * held. */ #define PGA_WRITEABLE 0x0001 /* page may be mapped writeable */ #define PGA_REFERENCED 0x0002 /* page has been referenced */ #define PGA_EXECUTABLE 0x0004 /* page may be mapped executable */ #define PGA_ENQUEUED 0x0008 /* page is enqueued in a page queue */ #define PGA_DEQUEUE 0x0010 /* page is due to be dequeued */ #define PGA_REQUEUE 0x0020 /* page is due to be requeued */ #define PGA_REQUEUE_HEAD 0x0040 /* page requeue should bypass LRU */ #define PGA_NOSYNC 0x0080 /* do not collect for syncer */ #define PGA_SWAP_FREE 0x0100 /* page with swap space was dirtied */ #define PGA_SWAP_SPACE 0x0200 /* page has allocated swap space */ #define PGA_QUEUE_OP_MASK (PGA_DEQUEUE | PGA_REQUEUE | PGA_REQUEUE_HEAD) #define PGA_QUEUE_STATE_MASK (PGA_ENQUEUED | PGA_QUEUE_OP_MASK) /* * Page flags. Updates to these flags are not synchronized, and thus they must * be set during page allocation or free to avoid races. * * The PG_PCPU_CACHE flag is set at allocation time if the page was * allocated from a per-CPU cache. It is cleared the next time that the * page is allocated from the physical memory allocator. */ #define PG_PCPU_CACHE 0x01 /* was allocated from per-CPU caches */ #define PG_FICTITIOUS 0x02 /* physical page doesn't exist */ #define PG_ZERO 0x04 /* page is zeroed */ #define PG_MARKER 0x08 /* special queue marker page */ #define PG_NODUMP 0x10 /* don't include this page in a dump */ /* * Misc constants. */ #define ACT_DECLINE 1 #define ACT_ADVANCE 3 #define ACT_INIT 5 #define ACT_MAX 64 #ifdef _KERNEL #include #include /* * Each pageable resident page falls into one of five lists: * * free * Available for allocation now. * * inactive * Low activity, candidates for reclamation. * This list is approximately LRU ordered. * * laundry * This is the list of pages that should be * paged out next. * * unswappable * Dirty anonymous pages that cannot be paged * out because no swap device is configured. * * active * Pages that are "active", i.e., they have been * recently referenced. * */ extern vm_page_t vm_page_array; /* First resident page in table */ extern long vm_page_array_size; /* number of vm_page_t's */ extern long first_page; /* first physical page number */ #define VM_PAGE_TO_PHYS(entry) ((entry)->phys_addr) /* * PHYS_TO_VM_PAGE() returns the vm_page_t object that represents a memory * page to which the given physical address belongs. The correct vm_page_t * object is returned for addresses that are not page-aligned. */ vm_page_t PHYS_TO_VM_PAGE(vm_paddr_t pa); /* * Page allocation parameters for vm_page for the functions * vm_page_alloc(), vm_page_grab(), vm_page_alloc_contig() and * vm_page_alloc_freelist(). Some functions support only a subset * of the flags, and ignore others, see the flags legend. * * The meaning of VM_ALLOC_ZERO differs slightly between the vm_page_alloc*() * and the vm_page_grab*() functions. See these functions for details. * * Bits 0 - 1 define class. * Bits 2 - 15 dedicated for flags. * Legend: * (a) - vm_page_alloc() supports the flag. * (c) - vm_page_alloc_contig() supports the flag. * (g) - vm_page_grab() supports the flag. * (n) - vm_page_alloc_noobj() and vm_page_alloc_freelist() support the flag. * (p) - vm_page_grab_pages() supports the flag. * Bits above 15 define the count of additional pages that the caller * intends to allocate. */ #define VM_ALLOC_NORMAL 0 #define VM_ALLOC_INTERRUPT 1 #define VM_ALLOC_SYSTEM 2 #define VM_ALLOC_CLASS_MASK 3 #define VM_ALLOC_WAITOK 0x0008 /* (acn) Sleep and retry */ #define VM_ALLOC_WAITFAIL 0x0010 /* (acn) Sleep and return error */ #define VM_ALLOC_WIRED 0x0020 /* (acgnp) Allocate a wired page */ #define VM_ALLOC_ZERO 0x0040 /* (acgnp) Allocate a zeroed page */ #define VM_ALLOC_NORECLAIM 0x0080 /* (c) Do not reclaim after failure */ #define VM_ALLOC_AVAIL0 0x0100 #define VM_ALLOC_NOBUSY 0x0200 /* (acgp) Do not excl busy the page */ #define VM_ALLOC_NOCREAT 0x0400 /* (gp) Don't create a page */ #define VM_ALLOC_AVAIL1 0x0800 #define VM_ALLOC_IGN_SBUSY 0x1000 /* (gp) Ignore shared busy flag */ #define VM_ALLOC_NODUMP 0x2000 /* (ag) don't include in dump */ #define VM_ALLOC_SBUSY 0x4000 /* (acgp) Shared busy the page */ #define VM_ALLOC_NOWAIT 0x8000 /* (acgnp) Do not sleep */ #define VM_ALLOC_COUNT_MAX 0xffff #define VM_ALLOC_COUNT_SHIFT 16 #define VM_ALLOC_COUNT_MASK (VM_ALLOC_COUNT(VM_ALLOC_COUNT_MAX)) #define VM_ALLOC_COUNT(count) ({ \ KASSERT((count) <= VM_ALLOC_COUNT_MAX, \ ("%s: invalid VM_ALLOC_COUNT value", __func__)); \ (count) << VM_ALLOC_COUNT_SHIFT; \ }) #ifdef M_NOWAIT static inline int malloc2vm_flags(int malloc_flags) { int pflags; KASSERT((malloc_flags & M_USE_RESERVE) == 0 || (malloc_flags & M_NOWAIT) != 0, ("M_USE_RESERVE requires M_NOWAIT")); pflags = (malloc_flags & M_USE_RESERVE) != 0 ? VM_ALLOC_INTERRUPT : VM_ALLOC_SYSTEM; if ((malloc_flags & M_ZERO) != 0) pflags |= VM_ALLOC_ZERO; if ((malloc_flags & M_NODUMP) != 0) pflags |= VM_ALLOC_NODUMP; if ((malloc_flags & M_NOWAIT)) pflags |= VM_ALLOC_NOWAIT; if ((malloc_flags & M_WAITOK)) pflags |= VM_ALLOC_WAITOK; if ((malloc_flags & M_NORECLAIM)) pflags |= VM_ALLOC_NORECLAIM; return (pflags); } #endif /* * Predicates supported by vm_page_ps_test(): * * PS_ALL_DIRTY is true only if the entire (super)page is dirty. * However, it can be spuriously false when the (super)page has become * dirty in the pmap but that information has not been propagated to the * machine-independent layer. */ #define PS_ALL_DIRTY 0x1 #define PS_ALL_VALID 0x2 #define PS_NONE_BUSY 0x4 bool vm_page_busy_acquire(vm_page_t m, int allocflags); void vm_page_busy_downgrade(vm_page_t m); int vm_page_busy_tryupgrade(vm_page_t m); bool vm_page_busy_sleep(vm_page_t m, const char *msg, int allocflags); void vm_page_busy_sleep_unlocked(vm_object_t obj, vm_page_t m, vm_pindex_t pindex, const char *wmesg, int allocflags); void vm_page_free(vm_page_t m); void vm_page_free_zero(vm_page_t m); void vm_page_activate (vm_page_t); void vm_page_advise(vm_page_t m, int advice); vm_page_t vm_page_alloc(vm_object_t, vm_pindex_t, int); vm_page_t vm_page_alloc_domain(vm_object_t, vm_pindex_t, int, int); vm_page_t vm_page_alloc_after(vm_object_t, vm_pindex_t, int, vm_page_t); vm_page_t vm_page_alloc_domain_after(vm_object_t, vm_pindex_t, int, int, vm_page_t); vm_page_t vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary, vm_memattr_t memattr); vm_page_t vm_page_alloc_contig_domain(vm_object_t object, vm_pindex_t pindex, int domain, int req, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary, vm_memattr_t memattr); vm_page_t vm_page_alloc_freelist(int, int); vm_page_t vm_page_alloc_freelist_domain(int, int, int); vm_page_t vm_page_alloc_noobj(int); vm_page_t vm_page_alloc_noobj_domain(int, int); vm_page_t vm_page_alloc_noobj_contig(int req, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary, vm_memattr_t memattr); vm_page_t vm_page_alloc_noobj_contig_domain(int domain, int req, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary, vm_memattr_t memattr); void vm_page_bits_set(vm_page_t m, vm_page_bits_t *bits, vm_page_bits_t set); bool vm_page_blacklist_add(vm_paddr_t pa, bool verbose); vm_page_t vm_page_grab(vm_object_t, vm_pindex_t, int); vm_page_t vm_page_grab_unlocked(vm_object_t, vm_pindex_t, int); int vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags, vm_page_t *ma, int count); int vm_page_grab_pages_unlocked(vm_object_t object, vm_pindex_t pindex, int allocflags, vm_page_t *ma, int count); int vm_page_grab_valid(vm_page_t *mp, vm_object_t object, vm_pindex_t pindex, int allocflags); int vm_page_grab_valid_unlocked(vm_page_t *mp, vm_object_t object, vm_pindex_t pindex, int allocflags); void vm_page_deactivate(vm_page_t); void vm_page_deactivate_noreuse(vm_page_t); void vm_page_dequeue(vm_page_t m); void vm_page_dequeue_deferred(vm_page_t m); vm_page_t vm_page_find_least(vm_object_t, vm_pindex_t); void vm_page_free_invalid(vm_page_t); vm_page_t vm_page_getfake(vm_paddr_t paddr, vm_memattr_t memattr); void vm_page_initfake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr); void vm_page_init_marker(vm_page_t marker, int queue, uint16_t aflags); void vm_page_init_page(vm_page_t m, vm_paddr_t pa, int segind); int vm_page_insert (vm_page_t, vm_object_t, vm_pindex_t); void vm_page_invalid(vm_page_t m); void vm_page_launder(vm_page_t m); vm_page_t vm_page_lookup(vm_object_t, vm_pindex_t); vm_page_t vm_page_lookup_unlocked(vm_object_t, vm_pindex_t); vm_page_t vm_page_next(vm_page_t m); void vm_page_pqbatch_drain(void); void vm_page_pqbatch_submit(vm_page_t m, uint8_t queue); bool vm_page_pqstate_commit(vm_page_t m, vm_page_astate_t *old, vm_page_astate_t new); vm_page_t vm_page_prev(vm_page_t m); bool vm_page_ps_test(vm_page_t m, int flags, vm_page_t skip_m); void vm_page_putfake(vm_page_t m); void vm_page_readahead_finish(vm_page_t m); int vm_page_reclaim_contig(int req, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary); int vm_page_reclaim_contig_domain(int domain, int req, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary); int vm_page_reclaim_contig_domain_ext(int domain, int req, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary, int desired_runs); void vm_page_reference(vm_page_t m); #define VPR_TRYFREE 0x01 #define VPR_NOREUSE 0x02 void vm_page_release(vm_page_t m, int flags); void vm_page_release_locked(vm_page_t m, int flags); vm_page_t vm_page_relookup(vm_object_t, vm_pindex_t); bool vm_page_remove(vm_page_t); bool vm_page_remove_xbusy(vm_page_t); int vm_page_rename(vm_page_t, vm_object_t, vm_pindex_t); void vm_page_replace(vm_page_t mnew, vm_object_t object, vm_pindex_t pindex, vm_page_t mold); int vm_page_sbusied(vm_page_t m); vm_page_bits_t vm_page_set_dirty(vm_page_t m); void vm_page_set_valid_range(vm_page_t m, int base, int size); vm_offset_t vm_page_startup(vm_offset_t vaddr); void vm_page_sunbusy(vm_page_t m); bool vm_page_try_remove_all(vm_page_t m); bool vm_page_try_remove_write(vm_page_t m); int vm_page_trysbusy(vm_page_t m); int vm_page_tryxbusy(vm_page_t m); void vm_page_unhold_pages(vm_page_t *ma, int count); void vm_page_unswappable(vm_page_t m); void vm_page_unwire(vm_page_t m, uint8_t queue); bool vm_page_unwire_noq(vm_page_t m); void vm_page_updatefake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr); void vm_page_wire(vm_page_t); bool vm_page_wire_mapped(vm_page_t m); void vm_page_xunbusy_hard(vm_page_t m); void vm_page_xunbusy_hard_unchecked(vm_page_t m); void vm_page_set_validclean (vm_page_t, int, int); void vm_page_clear_dirty(vm_page_t, int, int); void vm_page_set_invalid(vm_page_t, int, int); void vm_page_valid(vm_page_t m); int vm_page_is_valid(vm_page_t, int, int); void vm_page_test_dirty(vm_page_t); vm_page_bits_t vm_page_bits(int base, int size); void vm_page_zero_invalid(vm_page_t m, boolean_t setvalid); void vm_page_free_pages_toq(struct spglist *free, bool update_wire_count); void vm_page_dirty_KBI(vm_page_t m); void vm_page_lock_KBI(vm_page_t m, const char *file, int line); void vm_page_unlock_KBI(vm_page_t m, const char *file, int line); int vm_page_trylock_KBI(vm_page_t m, const char *file, int line); #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) void vm_page_assert_locked_KBI(vm_page_t m, const char *file, int line); void vm_page_lock_assert_KBI(vm_page_t m, int a, const char *file, int line); #endif #define vm_page_busy_fetch(m) atomic_load_int(&(m)->busy_lock) #define vm_page_assert_busied(m) \ KASSERT(vm_page_busied(m), \ ("vm_page_assert_busied: page %p not busy @ %s:%d", \ (m), __FILE__, __LINE__)) #define vm_page_assert_sbusied(m) \ KASSERT(vm_page_sbusied(m), \ ("vm_page_assert_sbusied: page %p not shared busy @ %s:%d", \ (m), __FILE__, __LINE__)) #define vm_page_assert_unbusied(m) \ KASSERT((vm_page_busy_fetch(m) & ~VPB_BIT_WAITERS) != \ VPB_CURTHREAD_EXCLUSIVE, \ ("vm_page_assert_unbusied: page %p busy_lock %#x owned" \ " by me (%p) @ %s:%d", \ (m), (m)->busy_lock, curthread, __FILE__, __LINE__)); \ #define vm_page_assert_xbusied_unchecked(m) do { \ KASSERT(vm_page_xbusied(m), \ ("vm_page_assert_xbusied: page %p not exclusive busy @ %s:%d", \ (m), __FILE__, __LINE__)); \ } while (0) #define vm_page_assert_xbusied(m) do { \ vm_page_assert_xbusied_unchecked(m); \ KASSERT((vm_page_busy_fetch(m) & ~VPB_BIT_WAITERS) == \ VPB_CURTHREAD_EXCLUSIVE, \ ("vm_page_assert_xbusied: page %p busy_lock %#x not owned" \ " by me (%p) @ %s:%d", \ (m), (m)->busy_lock, curthread, __FILE__, __LINE__)); \ } while (0) #define vm_page_busied(m) \ (vm_page_busy_fetch(m) != VPB_UNBUSIED) #define vm_page_xbusied(m) \ ((vm_page_busy_fetch(m) & VPB_SINGLE_EXCLUSIVE) != 0) #define vm_page_busy_freed(m) \ (vm_page_busy_fetch(m) == VPB_FREED) /* Note: page m's lock must not be owned by the caller. */ #define vm_page_xunbusy(m) do { \ if (!atomic_cmpset_rel_int(&(m)->busy_lock, \ VPB_CURTHREAD_EXCLUSIVE, VPB_UNBUSIED)) \ vm_page_xunbusy_hard(m); \ } while (0) #define vm_page_xunbusy_unchecked(m) do { \ if (!atomic_cmpset_rel_int(&(m)->busy_lock, \ VPB_CURTHREAD_EXCLUSIVE, VPB_UNBUSIED)) \ vm_page_xunbusy_hard_unchecked(m); \ } while (0) #ifdef INVARIANTS void vm_page_object_busy_assert(vm_page_t m); #define VM_PAGE_OBJECT_BUSY_ASSERT(m) vm_page_object_busy_assert(m) void vm_page_assert_pga_writeable(vm_page_t m, uint16_t bits); #define VM_PAGE_ASSERT_PGA_WRITEABLE(m, bits) \ vm_page_assert_pga_writeable(m, bits) /* * Claim ownership of a page's xbusy state. In non-INVARIANTS kernels this * operation is a no-op since ownership is not tracked. In particular * this macro does not provide any synchronization with the previous owner. */ #define vm_page_xbusy_claim(m) do { \ u_int _busy_lock; \ \ vm_page_assert_xbusied_unchecked((m)); \ do { \ _busy_lock = vm_page_busy_fetch(m); \ } while (!atomic_cmpset_int(&(m)->busy_lock, _busy_lock, \ (_busy_lock & VPB_BIT_FLAGMASK) | VPB_CURTHREAD_EXCLUSIVE)); \ } while (0) #else #define VM_PAGE_OBJECT_BUSY_ASSERT(m) (void)0 #define VM_PAGE_ASSERT_PGA_WRITEABLE(m, bits) (void)0 #define vm_page_xbusy_claim(m) #endif #if BYTE_ORDER == BIG_ENDIAN #define VM_PAGE_AFLAG_SHIFT 16 #else #define VM_PAGE_AFLAG_SHIFT 0 #endif /* * Load a snapshot of a page's 32-bit atomic state. */ static inline vm_page_astate_t vm_page_astate_load(vm_page_t m) { vm_page_astate_t a; a._bits = atomic_load_32(&m->a._bits); return (a); } /* * Atomically compare and set a page's atomic state. */ static inline bool vm_page_astate_fcmpset(vm_page_t m, vm_page_astate_t *old, vm_page_astate_t new) { KASSERT(new.queue == PQ_INACTIVE || (new.flags & PGA_REQUEUE_HEAD) == 0, ("%s: invalid head requeue request for page %p", __func__, m)); KASSERT((new.flags & PGA_ENQUEUED) == 0 || new.queue != PQ_NONE, ("%s: setting PGA_ENQUEUED with PQ_NONE in page %p", __func__, m)); KASSERT(new._bits != old->_bits, ("%s: bits are unchanged", __func__)); return (atomic_fcmpset_32(&m->a._bits, &old->_bits, new._bits) != 0); } /* * Clear the given bits in the specified page. */ static inline void vm_page_aflag_clear(vm_page_t m, uint16_t bits) { uint32_t *addr, val; /* * Access the whole 32-bit word containing the aflags field with an * atomic update. Parallel non-atomic updates to the other fields * within this word are handled properly by the atomic update. */ addr = (void *)&m->a; val = bits << VM_PAGE_AFLAG_SHIFT; atomic_clear_32(addr, val); } /* * Set the given bits in the specified page. */ static inline void vm_page_aflag_set(vm_page_t m, uint16_t bits) { uint32_t *addr, val; VM_PAGE_ASSERT_PGA_WRITEABLE(m, bits); /* * Access the whole 32-bit word containing the aflags field with an * atomic update. Parallel non-atomic updates to the other fields * within this word are handled properly by the atomic update. */ addr = (void *)&m->a; val = bits << VM_PAGE_AFLAG_SHIFT; atomic_set_32(addr, val); } /* * vm_page_dirty: * * Set all bits in the page's dirty field. * * The object containing the specified page must be locked if the * call is made from the machine-independent layer. * * See vm_page_clear_dirty_mask(). */ static __inline void vm_page_dirty(vm_page_t m) { /* Use vm_page_dirty_KBI() under INVARIANTS to save memory. */ #if (defined(KLD_MODULE) && !defined(KLD_TIED)) || defined(INVARIANTS) vm_page_dirty_KBI(m); #else m->dirty = VM_PAGE_BITS_ALL; #endif } /* * vm_page_undirty: * * Set page to not be dirty. Note: does not clear pmap modify bits */ static __inline void vm_page_undirty(vm_page_t m) { VM_PAGE_OBJECT_BUSY_ASSERT(m); m->dirty = 0; } static inline uint8_t _vm_page_queue(vm_page_astate_t as) { if ((as.flags & PGA_DEQUEUE) != 0) return (PQ_NONE); return (as.queue); } /* * vm_page_queue: * * Return the index of the queue containing m. */ static inline uint8_t vm_page_queue(vm_page_t m) { return (_vm_page_queue(vm_page_astate_load(m))); } static inline bool vm_page_active(vm_page_t m) { return (vm_page_queue(m) == PQ_ACTIVE); } static inline bool vm_page_inactive(vm_page_t m) { return (vm_page_queue(m) == PQ_INACTIVE); } static inline bool vm_page_in_laundry(vm_page_t m) { uint8_t queue; queue = vm_page_queue(m); return (queue == PQ_LAUNDRY || queue == PQ_UNSWAPPABLE); } /* * vm_page_drop: * * Release a reference to a page and return the old reference count. */ static inline u_int vm_page_drop(vm_page_t m, u_int val) { u_int old; /* * Synchronize with vm_page_free_prep(): ensure that all updates to the * page structure are visible before it is freed. */ atomic_thread_fence_rel(); old = atomic_fetchadd_int(&m->ref_count, -val); KASSERT(old != VPRC_BLOCKED, ("vm_page_drop: page %p has an invalid refcount value", m)); return (old); } /* * vm_page_wired: * * Perform a racy check to determine whether a reference prevents the page * from being reclaimable. If the page's object is locked, and the page is * unmapped and exclusively busied by the current thread, no new wirings * may be created. */ static inline bool vm_page_wired(vm_page_t m) { return (VPRC_WIRE_COUNT(m->ref_count) > 0); } static inline bool vm_page_all_valid(vm_page_t m) { return (m->valid == VM_PAGE_BITS_ALL); } static inline bool vm_page_any_valid(vm_page_t m) { return (m->valid != 0); } static inline bool vm_page_none_valid(vm_page_t m) { return (m->valid == 0); } static inline int -vm_page_domain(vm_page_t m) +vm_page_domain(vm_page_t m __numa_used) { #ifdef NUMA int domn, segind; segind = m->segind; KASSERT(segind < vm_phys_nsegs, ("segind %d m %p", segind, m)); domn = vm_phys_segs[segind].domain; KASSERT(domn >= 0 && domn < vm_ndomains, ("domain %d m %p", domn, m)); return (domn); #else return (0); #endif } #endif /* _KERNEL */ #endif /* !_VM_PAGE_ */ diff --git a/sys/vm/vm_phys.c b/sys/vm/vm_phys.c index 8c15f107c2f9..66ad9c13dceb 100644 --- a/sys/vm/vm_phys.c +++ b/sys/vm/vm_phys.c @@ -1,1862 +1,1859 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2002-2006 Rice University * Copyright (c) 2007 Alan L. Cox * All rights reserved. * * This software was developed for the FreeBSD Project by Alan L. Cox, * Olivier Crameri, Peter Druschel, Sitaram Iyer, and Juan Navarro. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Physical memory system implementation * * Any external functions defined by this module are only to be used by the * virtual memory system. */ #include #include "opt_ddb.h" #include "opt_vm.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include _Static_assert(sizeof(long) * NBBY >= VM_PHYSSEG_MAX, "Too many physsegs."); _Static_assert(sizeof(long long) >= sizeof(vm_paddr_t), "vm_paddr_t too big for ffsll, flsll."); #ifdef NUMA struct mem_affinity __read_mostly *mem_affinity; int __read_mostly *mem_locality; static int numa_disabled; static SYSCTL_NODE(_vm, OID_AUTO, numa, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "NUMA options"); SYSCTL_INT(_vm_numa, OID_AUTO, disabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &numa_disabled, 0, "NUMA-awareness in the allocators is disabled"); #endif int __read_mostly vm_ndomains = 1; domainset_t __read_mostly all_domains = DOMAINSET_T_INITIALIZER(0x1); struct vm_phys_seg __read_mostly vm_phys_segs[VM_PHYSSEG_MAX]; int __read_mostly vm_phys_nsegs; static struct vm_phys_seg vm_phys_early_segs[8]; static int vm_phys_early_nsegs; struct vm_phys_fictitious_seg; static int vm_phys_fictitious_cmp(struct vm_phys_fictitious_seg *, struct vm_phys_fictitious_seg *); RB_HEAD(fict_tree, vm_phys_fictitious_seg) vm_phys_fictitious_tree = RB_INITIALIZER(&vm_phys_fictitious_tree); struct vm_phys_fictitious_seg { RB_ENTRY(vm_phys_fictitious_seg) node; /* Memory region data */ vm_paddr_t start; vm_paddr_t end; vm_page_t first_page; }; RB_GENERATE_STATIC(fict_tree, vm_phys_fictitious_seg, node, vm_phys_fictitious_cmp); static struct rwlock_padalign vm_phys_fictitious_reg_lock; MALLOC_DEFINE(M_FICT_PAGES, "vm_fictitious", "Fictitious VM pages"); static struct vm_freelist __aligned(CACHE_LINE_SIZE) vm_phys_free_queues[MAXMEMDOM][VM_NFREELIST][VM_NFREEPOOL] [VM_NFREEORDER_MAX]; static int __read_mostly vm_nfreelists; /* * These "avail lists" are globals used to communicate boot-time physical * memory layout to other parts of the kernel. Each physically contiguous * region of memory is defined by a start address at an even index and an * end address at the following odd index. Each list is terminated by a * pair of zero entries. * * dump_avail tells the dump code what regions to include in a crash dump, and * phys_avail is all of the remaining physical memory that is available for * the vm system. * * Initially dump_avail and phys_avail are identical. Boot time memory * allocations remove extents from phys_avail that may still be included * in dumps. */ vm_paddr_t phys_avail[PHYS_AVAIL_COUNT]; vm_paddr_t dump_avail[PHYS_AVAIL_COUNT]; /* * Provides the mapping from VM_FREELIST_* to free list indices (flind). */ static int __read_mostly vm_freelist_to_flind[VM_NFREELIST]; CTASSERT(VM_FREELIST_DEFAULT == 0); #ifdef VM_FREELIST_DMA32 #define VM_DMA32_BOUNDARY ((vm_paddr_t)1 << 32) #endif /* * Enforce the assumptions made by vm_phys_add_seg() and vm_phys_init() about * the ordering of the free list boundaries. */ #if defined(VM_LOWMEM_BOUNDARY) && defined(VM_DMA32_BOUNDARY) CTASSERT(VM_LOWMEM_BOUNDARY < VM_DMA32_BOUNDARY); #endif static int sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS); SYSCTL_OID(_vm, OID_AUTO, phys_free, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_phys_free, "A", "Phys Free Info"); static int sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS); SYSCTL_OID(_vm, OID_AUTO, phys_segs, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_phys_segs, "A", "Phys Seg Info"); #ifdef NUMA static int sysctl_vm_phys_locality(SYSCTL_HANDLER_ARGS); SYSCTL_OID(_vm, OID_AUTO, phys_locality, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_phys_locality, "A", "Phys Locality Info"); #endif SYSCTL_INT(_vm, OID_AUTO, ndomains, CTLFLAG_RD, &vm_ndomains, 0, "Number of physical memory domains available."); static void _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int domain); static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end); static void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, int order, int tail); /* * Red-black tree helpers for vm fictitious range management. */ static inline int vm_phys_fictitious_in_range(struct vm_phys_fictitious_seg *p, struct vm_phys_fictitious_seg *range) { KASSERT(range->start != 0 && range->end != 0, ("Invalid range passed on search for vm_fictitious page")); if (p->start >= range->end) return (1); if (p->start < range->start) return (-1); return (0); } static int vm_phys_fictitious_cmp(struct vm_phys_fictitious_seg *p1, struct vm_phys_fictitious_seg *p2) { /* Check if this is a search for a page */ if (p1->end == 0) return (vm_phys_fictitious_in_range(p1, p2)); KASSERT(p2->end != 0, ("Invalid range passed as second parameter to vm fictitious comparison")); /* Searching to add a new range */ if (p1->end <= p2->start) return (-1); if (p1->start >= p2->end) return (1); panic("Trying to add overlapping vm fictitious ranges:\n" "[%#jx:%#jx] and [%#jx:%#jx]", (uintmax_t)p1->start, (uintmax_t)p1->end, (uintmax_t)p2->start, (uintmax_t)p2->end); } int -vm_phys_domain_match(int prefer, vm_paddr_t low, vm_paddr_t high) +vm_phys_domain_match(int prefer __numa_used, vm_paddr_t low __numa_used, + vm_paddr_t high __numa_used) { #ifdef NUMA domainset_t mask; int i; if (vm_ndomains == 1 || mem_affinity == NULL) return (0); DOMAINSET_ZERO(&mask); /* * Check for any memory that overlaps low, high. */ for (i = 0; mem_affinity[i].end != 0; i++) if (mem_affinity[i].start <= high && mem_affinity[i].end >= low) DOMAINSET_SET(mem_affinity[i].domain, &mask); if (prefer != -1 && DOMAINSET_ISSET(prefer, &mask)) return (prefer); if (DOMAINSET_EMPTY(&mask)) panic("vm_phys_domain_match: Impossible constraint"); return (DOMAINSET_FFS(&mask) - 1); #else return (0); #endif } /* * Outputs the state of the physical memory allocator, specifically, * the amount of physical memory in each free list. */ static int sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS) { struct sbuf sbuf; struct vm_freelist *fl; int dom, error, flind, oind, pind; error = sysctl_wire_old_buffer(req, 0); if (error != 0) return (error); sbuf_new_for_sysctl(&sbuf, NULL, 128 * vm_ndomains, req); for (dom = 0; dom < vm_ndomains; dom++) { sbuf_printf(&sbuf,"\nDOMAIN %d:\n", dom); for (flind = 0; flind < vm_nfreelists; flind++) { sbuf_printf(&sbuf, "\nFREE LIST %d:\n" "\n ORDER (SIZE) | NUMBER" "\n ", flind); for (pind = 0; pind < VM_NFREEPOOL; pind++) sbuf_printf(&sbuf, " | POOL %d", pind); sbuf_printf(&sbuf, "\n-- "); for (pind = 0; pind < VM_NFREEPOOL; pind++) sbuf_printf(&sbuf, "-- -- "); sbuf_printf(&sbuf, "--\n"); for (oind = VM_NFREEORDER - 1; oind >= 0; oind--) { sbuf_printf(&sbuf, " %2d (%6dK)", oind, 1 << (PAGE_SHIFT - 10 + oind)); for (pind = 0; pind < VM_NFREEPOOL; pind++) { fl = vm_phys_free_queues[dom][flind][pind]; sbuf_printf(&sbuf, " | %6d", fl[oind].lcnt); } sbuf_printf(&sbuf, "\n"); } } } error = sbuf_finish(&sbuf); sbuf_delete(&sbuf); return (error); } /* * Outputs the set of physical memory segments. */ static int sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS) { struct sbuf sbuf; struct vm_phys_seg *seg; int error, segind; error = sysctl_wire_old_buffer(req, 0); if (error != 0) return (error); sbuf_new_for_sysctl(&sbuf, NULL, 128, req); for (segind = 0; segind < vm_phys_nsegs; segind++) { sbuf_printf(&sbuf, "\nSEGMENT %d:\n\n", segind); seg = &vm_phys_segs[segind]; sbuf_printf(&sbuf, "start: %#jx\n", (uintmax_t)seg->start); sbuf_printf(&sbuf, "end: %#jx\n", (uintmax_t)seg->end); sbuf_printf(&sbuf, "domain: %d\n", seg->domain); sbuf_printf(&sbuf, "free list: %p\n", seg->free_queues); } error = sbuf_finish(&sbuf); sbuf_delete(&sbuf); return (error); } /* * Return affinity, or -1 if there's no affinity information. */ int -vm_phys_mem_affinity(int f, int t) +vm_phys_mem_affinity(int f __numa_used, int t __numa_used) { #ifdef NUMA if (mem_locality == NULL) return (-1); if (f >= vm_ndomains || t >= vm_ndomains) return (-1); return (mem_locality[f * vm_ndomains + t]); #else return (-1); #endif } #ifdef NUMA /* * Outputs the VM locality table. */ static int sysctl_vm_phys_locality(SYSCTL_HANDLER_ARGS) { struct sbuf sbuf; int error, i, j; error = sysctl_wire_old_buffer(req, 0); if (error != 0) return (error); sbuf_new_for_sysctl(&sbuf, NULL, 128, req); sbuf_printf(&sbuf, "\n"); for (i = 0; i < vm_ndomains; i++) { sbuf_printf(&sbuf, "%d: ", i); for (j = 0; j < vm_ndomains; j++) { sbuf_printf(&sbuf, "%d ", vm_phys_mem_affinity(i, j)); } sbuf_printf(&sbuf, "\n"); } error = sbuf_finish(&sbuf); sbuf_delete(&sbuf); return (error); } #endif static void vm_freelist_add(struct vm_freelist *fl, vm_page_t m, int order, int tail) { m->order = order; if (tail) TAILQ_INSERT_TAIL(&fl[order].pl, m, listq); else TAILQ_INSERT_HEAD(&fl[order].pl, m, listq); fl[order].lcnt++; } static void vm_freelist_rem(struct vm_freelist *fl, vm_page_t m, int order) { TAILQ_REMOVE(&fl[order].pl, m, listq); fl[order].lcnt--; m->order = VM_NFREEORDER; } /* * Create a physical memory segment. */ static void _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int domain) { struct vm_phys_seg *seg; KASSERT(vm_phys_nsegs < VM_PHYSSEG_MAX, ("vm_phys_create_seg: increase VM_PHYSSEG_MAX")); KASSERT(domain >= 0 && domain < vm_ndomains, ("vm_phys_create_seg: invalid domain provided")); seg = &vm_phys_segs[vm_phys_nsegs++]; while (seg > vm_phys_segs && (seg - 1)->start >= end) { *seg = *(seg - 1); seg--; } seg->start = start; seg->end = end; seg->domain = domain; } static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end) { #ifdef NUMA int i; if (mem_affinity == NULL) { _vm_phys_create_seg(start, end, 0); return; } for (i = 0;; i++) { if (mem_affinity[i].end == 0) panic("Reached end of affinity info"); if (mem_affinity[i].end <= start) continue; if (mem_affinity[i].start > start) panic("No affinity info for start %jx", (uintmax_t)start); if (mem_affinity[i].end >= end) { _vm_phys_create_seg(start, end, mem_affinity[i].domain); break; } _vm_phys_create_seg(start, mem_affinity[i].end, mem_affinity[i].domain); start = mem_affinity[i].end; } #else _vm_phys_create_seg(start, end, 0); #endif } /* * Add a physical memory segment. */ void vm_phys_add_seg(vm_paddr_t start, vm_paddr_t end) { vm_paddr_t paddr; KASSERT((start & PAGE_MASK) == 0, ("vm_phys_define_seg: start is not page aligned")); KASSERT((end & PAGE_MASK) == 0, ("vm_phys_define_seg: end is not page aligned")); /* * Split the physical memory segment if it spans two or more free * list boundaries. */ paddr = start; #ifdef VM_FREELIST_LOWMEM if (paddr < VM_LOWMEM_BOUNDARY && end > VM_LOWMEM_BOUNDARY) { vm_phys_create_seg(paddr, VM_LOWMEM_BOUNDARY); paddr = VM_LOWMEM_BOUNDARY; } #endif #ifdef VM_FREELIST_DMA32 if (paddr < VM_DMA32_BOUNDARY && end > VM_DMA32_BOUNDARY) { vm_phys_create_seg(paddr, VM_DMA32_BOUNDARY); paddr = VM_DMA32_BOUNDARY; } #endif vm_phys_create_seg(paddr, end); } /* * Initialize the physical memory allocator. * * Requires that vm_page_array is initialized! */ void vm_phys_init(void) { struct vm_freelist *fl; struct vm_phys_seg *end_seg, *prev_seg, *seg, *tmp_seg; #if defined(VM_DMA32_NPAGES_THRESHOLD) || defined(VM_PHYSSEG_SPARSE) u_long npages; #endif int dom, flind, freelist, oind, pind, segind; /* * Compute the number of free lists, and generate the mapping from the * manifest constants VM_FREELIST_* to the free list indices. * * Initially, the entries of vm_freelist_to_flind[] are set to either * 0 or 1 to indicate which free lists should be created. */ #ifdef VM_DMA32_NPAGES_THRESHOLD npages = 0; #endif for (segind = vm_phys_nsegs - 1; segind >= 0; segind--) { seg = &vm_phys_segs[segind]; #ifdef VM_FREELIST_LOWMEM if (seg->end <= VM_LOWMEM_BOUNDARY) vm_freelist_to_flind[VM_FREELIST_LOWMEM] = 1; else #endif #ifdef VM_FREELIST_DMA32 if ( #ifdef VM_DMA32_NPAGES_THRESHOLD /* * Create the DMA32 free list only if the amount of * physical memory above physical address 4G exceeds the * given threshold. */ npages > VM_DMA32_NPAGES_THRESHOLD && #endif seg->end <= VM_DMA32_BOUNDARY) vm_freelist_to_flind[VM_FREELIST_DMA32] = 1; else #endif { #ifdef VM_DMA32_NPAGES_THRESHOLD npages += atop(seg->end - seg->start); #endif vm_freelist_to_flind[VM_FREELIST_DEFAULT] = 1; } } /* Change each entry into a running total of the free lists. */ for (freelist = 1; freelist < VM_NFREELIST; freelist++) { vm_freelist_to_flind[freelist] += vm_freelist_to_flind[freelist - 1]; } vm_nfreelists = vm_freelist_to_flind[VM_NFREELIST - 1]; KASSERT(vm_nfreelists > 0, ("vm_phys_init: no free lists")); /* Change each entry into a free list index. */ for (freelist = 0; freelist < VM_NFREELIST; freelist++) vm_freelist_to_flind[freelist]--; /* * Initialize the first_page and free_queues fields of each physical * memory segment. */ #ifdef VM_PHYSSEG_SPARSE npages = 0; #endif for (segind = 0; segind < vm_phys_nsegs; segind++) { seg = &vm_phys_segs[segind]; #ifdef VM_PHYSSEG_SPARSE seg->first_page = &vm_page_array[npages]; npages += atop(seg->end - seg->start); #else seg->first_page = PHYS_TO_VM_PAGE(seg->start); #endif #ifdef VM_FREELIST_LOWMEM if (seg->end <= VM_LOWMEM_BOUNDARY) { flind = vm_freelist_to_flind[VM_FREELIST_LOWMEM]; KASSERT(flind >= 0, ("vm_phys_init: LOWMEM flind < 0")); } else #endif #ifdef VM_FREELIST_DMA32 if (seg->end <= VM_DMA32_BOUNDARY) { flind = vm_freelist_to_flind[VM_FREELIST_DMA32]; KASSERT(flind >= 0, ("vm_phys_init: DMA32 flind < 0")); } else #endif { flind = vm_freelist_to_flind[VM_FREELIST_DEFAULT]; KASSERT(flind >= 0, ("vm_phys_init: DEFAULT flind < 0")); } seg->free_queues = &vm_phys_free_queues[seg->domain][flind]; } /* * Coalesce physical memory segments that are contiguous and share the * same per-domain free queues. */ prev_seg = vm_phys_segs; seg = &vm_phys_segs[1]; end_seg = &vm_phys_segs[vm_phys_nsegs]; while (seg < end_seg) { if (prev_seg->end == seg->start && prev_seg->free_queues == seg->free_queues) { prev_seg->end = seg->end; KASSERT(prev_seg->domain == seg->domain, ("vm_phys_init: free queues cannot span domains")); vm_phys_nsegs--; end_seg--; for (tmp_seg = seg; tmp_seg < end_seg; tmp_seg++) *tmp_seg = *(tmp_seg + 1); } else { prev_seg = seg; seg++; } } /* * Initialize the free queues. */ for (dom = 0; dom < vm_ndomains; dom++) { for (flind = 0; flind < vm_nfreelists; flind++) { for (pind = 0; pind < VM_NFREEPOOL; pind++) { fl = vm_phys_free_queues[dom][flind][pind]; for (oind = 0; oind < VM_NFREEORDER; oind++) TAILQ_INIT(&fl[oind].pl); } } } rw_init(&vm_phys_fictitious_reg_lock, "vmfctr"); } /* * Register info about the NUMA topology of the system. * * Invoked by platform-dependent code prior to vm_phys_init(). */ void -vm_phys_register_domains(int ndomains, struct mem_affinity *affinity, - int *locality) +vm_phys_register_domains(int ndomains __numa_used, + struct mem_affinity *affinity __numa_used, int *locality __numa_used) { #ifdef NUMA int i; /* * For now the only override value that we support is 1, which * effectively disables NUMA-awareness in the allocators. */ TUNABLE_INT_FETCH("vm.numa.disabled", &numa_disabled); if (numa_disabled) ndomains = 1; if (ndomains > 1) { vm_ndomains = ndomains; mem_affinity = affinity; mem_locality = locality; } for (i = 0; i < vm_ndomains; i++) DOMAINSET_SET(i, &all_domains); -#else - (void)ndomains; - (void)affinity; - (void)locality; #endif } /* * Split a contiguous, power of two-sized set of physical pages. * * When this function is called by a page allocation function, the caller * should request insertion at the head unless the order [order, oind) queues * are known to be empty. The objective being to reduce the likelihood of * long-term fragmentation by promoting contemporaneous allocation and * (hopefully) deallocation. */ static __inline void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, int order, int tail) { vm_page_t m_buddy; while (oind > order) { oind--; m_buddy = &m[1 << oind]; KASSERT(m_buddy->order == VM_NFREEORDER, ("vm_phys_split_pages: page %p has unexpected order %d", m_buddy, m_buddy->order)); vm_freelist_add(fl, m_buddy, oind, tail); } } /* * Add the physical pages [m, m + npages) at the beginning of a power-of-two * aligned and sized set to the specified free list. * * When this function is called by a page allocation function, the caller * should request insertion at the head unless the lower-order queues are * known to be empty. The objective being to reduce the likelihood of long- * term fragmentation by promoting contemporaneous allocation and (hopefully) * deallocation. * * The physical page m's buddy must not be free. */ static void vm_phys_enq_beg(vm_page_t m, u_int npages, struct vm_freelist *fl, int tail) { int order; KASSERT(npages == 0 || (VM_PAGE_TO_PHYS(m) & ((PAGE_SIZE << (fls(npages) - 1)) - 1)) == 0, ("%s: page %p and npages %u are misaligned", __func__, m, npages)); while (npages > 0) { KASSERT(m->order == VM_NFREEORDER, ("%s: page %p has unexpected order %d", __func__, m, m->order)); order = fls(npages) - 1; KASSERT(order < VM_NFREEORDER, ("%s: order %d is out of range", __func__, order)); vm_freelist_add(fl, m, order, tail); m += 1 << order; npages -= 1 << order; } } /* * Add the physical pages [m, m + npages) at the end of a power-of-two aligned * and sized set to the specified free list. * * When this function is called by a page allocation function, the caller * should request insertion at the head unless the lower-order queues are * known to be empty. The objective being to reduce the likelihood of long- * term fragmentation by promoting contemporaneous allocation and (hopefully) * deallocation. * * If npages is zero, this function does nothing and ignores the physical page * parameter m. Otherwise, the physical page m's buddy must not be free. */ static vm_page_t vm_phys_enq_range(vm_page_t m, u_int npages, struct vm_freelist *fl, int tail) { int order; KASSERT(npages == 0 || ((VM_PAGE_TO_PHYS(m) + npages * PAGE_SIZE) & ((PAGE_SIZE << (fls(npages) - 1)) - 1)) == 0, ("vm_phys_enq_range: page %p and npages %u are misaligned", m, npages)); while (npages > 0) { KASSERT(m->order == VM_NFREEORDER, ("vm_phys_enq_range: page %p has unexpected order %d", m, m->order)); order = ffs(npages) - 1; KASSERT(order < VM_NFREEORDER, ("vm_phys_enq_range: order %d is out of range", order)); vm_freelist_add(fl, m, order, tail); m += 1 << order; npages -= 1 << order; } return (m); } /* * Set the pool for a contiguous, power of two-sized set of physical pages. */ static void vm_phys_set_pool(int pool, vm_page_t m, int order) { vm_page_t m_tmp; for (m_tmp = m; m_tmp < &m[1 << order]; m_tmp++) m_tmp->pool = pool; } /* * Tries to allocate the specified number of pages from the specified pool * within the specified domain. Returns the actual number of allocated pages * and a pointer to each page through the array ma[]. * * The returned pages may not be physically contiguous. However, in contrast * to performing multiple, back-to-back calls to vm_phys_alloc_pages(..., 0), * calling this function once to allocate the desired number of pages will * avoid wasted time in vm_phys_split_pages(). * * The free page queues for the specified domain must be locked. */ int vm_phys_alloc_npages(int domain, int pool, int npages, vm_page_t ma[]) { struct vm_freelist *alt, *fl; vm_page_t m; int avail, end, flind, freelist, i, oind, pind; KASSERT(domain >= 0 && domain < vm_ndomains, ("vm_phys_alloc_npages: domain %d is out of range", domain)); KASSERT(pool < VM_NFREEPOOL, ("vm_phys_alloc_npages: pool %d is out of range", pool)); KASSERT(npages <= 1 << (VM_NFREEORDER - 1), ("vm_phys_alloc_npages: npages %d is out of range", npages)); vm_domain_free_assert_locked(VM_DOMAIN(domain)); i = 0; for (freelist = 0; freelist < VM_NFREELIST; freelist++) { flind = vm_freelist_to_flind[freelist]; if (flind < 0) continue; fl = vm_phys_free_queues[domain][flind][pool]; for (oind = 0; oind < VM_NFREEORDER; oind++) { while ((m = TAILQ_FIRST(&fl[oind].pl)) != NULL) { vm_freelist_rem(fl, m, oind); avail = i + (1 << oind); end = imin(npages, avail); while (i < end) ma[i++] = m++; if (i == npages) { /* * Return excess pages to fl. Its order * [0, oind) queues are empty. */ vm_phys_enq_range(m, avail - i, fl, 1); return (npages); } } } for (oind = VM_NFREEORDER - 1; oind >= 0; oind--) { for (pind = 0; pind < VM_NFREEPOOL; pind++) { alt = vm_phys_free_queues[domain][flind][pind]; while ((m = TAILQ_FIRST(&alt[oind].pl)) != NULL) { vm_freelist_rem(alt, m, oind); vm_phys_set_pool(pool, m, oind); avail = i + (1 << oind); end = imin(npages, avail); while (i < end) ma[i++] = m++; if (i == npages) { /* * Return excess pages to fl. * Its order [0, oind) queues * are empty. */ vm_phys_enq_range(m, avail - i, fl, 1); return (npages); } } } } } return (i); } /* * Allocate a contiguous, power of two-sized set of physical pages * from the free lists. * * The free page queues must be locked. */ vm_page_t vm_phys_alloc_pages(int domain, int pool, int order) { vm_page_t m; int freelist; for (freelist = 0; freelist < VM_NFREELIST; freelist++) { m = vm_phys_alloc_freelist_pages(domain, freelist, pool, order); if (m != NULL) return (m); } return (NULL); } /* * Allocate a contiguous, power of two-sized set of physical pages from the * specified free list. The free list must be specified using one of the * manifest constants VM_FREELIST_*. * * The free page queues must be locked. */ vm_page_t vm_phys_alloc_freelist_pages(int domain, int freelist, int pool, int order) { struct vm_freelist *alt, *fl; vm_page_t m; int oind, pind, flind; KASSERT(domain >= 0 && domain < vm_ndomains, ("vm_phys_alloc_freelist_pages: domain %d is out of range", domain)); KASSERT(freelist < VM_NFREELIST, ("vm_phys_alloc_freelist_pages: freelist %d is out of range", freelist)); KASSERT(pool < VM_NFREEPOOL, ("vm_phys_alloc_freelist_pages: pool %d is out of range", pool)); KASSERT(order < VM_NFREEORDER, ("vm_phys_alloc_freelist_pages: order %d is out of range", order)); flind = vm_freelist_to_flind[freelist]; /* Check if freelist is present */ if (flind < 0) return (NULL); vm_domain_free_assert_locked(VM_DOMAIN(domain)); fl = &vm_phys_free_queues[domain][flind][pool][0]; for (oind = order; oind < VM_NFREEORDER; oind++) { m = TAILQ_FIRST(&fl[oind].pl); if (m != NULL) { vm_freelist_rem(fl, m, oind); /* The order [order, oind) queues are empty. */ vm_phys_split_pages(m, oind, fl, order, 1); return (m); } } /* * The given pool was empty. Find the largest * contiguous, power-of-two-sized set of pages in any * pool. Transfer these pages to the given pool, and * use them to satisfy the allocation. */ for (oind = VM_NFREEORDER - 1; oind >= order; oind--) { for (pind = 0; pind < VM_NFREEPOOL; pind++) { alt = &vm_phys_free_queues[domain][flind][pind][0]; m = TAILQ_FIRST(&alt[oind].pl); if (m != NULL) { vm_freelist_rem(alt, m, oind); vm_phys_set_pool(pool, m, oind); /* The order [order, oind) queues are empty. */ vm_phys_split_pages(m, oind, fl, order, 1); return (m); } } } return (NULL); } /* * Find the vm_page corresponding to the given physical address. */ vm_page_t vm_phys_paddr_to_vm_page(vm_paddr_t pa) { struct vm_phys_seg *seg; if ((seg = vm_phys_paddr_to_seg(pa)) != NULL) return (&seg->first_page[atop(pa - seg->start)]); return (NULL); } vm_page_t vm_phys_fictitious_to_vm_page(vm_paddr_t pa) { struct vm_phys_fictitious_seg tmp, *seg; vm_page_t m; m = NULL; tmp.start = pa; tmp.end = 0; rw_rlock(&vm_phys_fictitious_reg_lock); seg = RB_FIND(fict_tree, &vm_phys_fictitious_tree, &tmp); rw_runlock(&vm_phys_fictitious_reg_lock); if (seg == NULL) return (NULL); m = &seg->first_page[atop(pa - seg->start)]; KASSERT((m->flags & PG_FICTITIOUS) != 0, ("%p not fictitious", m)); return (m); } static inline void vm_phys_fictitious_init_range(vm_page_t range, vm_paddr_t start, long page_count, vm_memattr_t memattr) { long i; bzero(range, page_count * sizeof(*range)); for (i = 0; i < page_count; i++) { vm_page_initfake(&range[i], start + PAGE_SIZE * i, memattr); range[i].oflags &= ~VPO_UNMANAGED; range[i].busy_lock = VPB_UNBUSIED; } } int vm_phys_fictitious_reg_range(vm_paddr_t start, vm_paddr_t end, vm_memattr_t memattr) { struct vm_phys_fictitious_seg *seg; vm_page_t fp; long page_count; #ifdef VM_PHYSSEG_DENSE long pi, pe; long dpage_count; #endif KASSERT(start < end, ("Start of segment isn't less than end (start: %jx end: %jx)", (uintmax_t)start, (uintmax_t)end)); page_count = (end - start) / PAGE_SIZE; #ifdef VM_PHYSSEG_DENSE pi = atop(start); pe = atop(end); if (pi >= first_page && (pi - first_page) < vm_page_array_size) { fp = &vm_page_array[pi - first_page]; if ((pe - first_page) > vm_page_array_size) { /* * We have a segment that starts inside * of vm_page_array, but ends outside of it. * * Use vm_page_array pages for those that are * inside of the vm_page_array range, and * allocate the remaining ones. */ dpage_count = vm_page_array_size - (pi - first_page); vm_phys_fictitious_init_range(fp, start, dpage_count, memattr); page_count -= dpage_count; start += ptoa(dpage_count); goto alloc; } /* * We can allocate the full range from vm_page_array, * so there's no need to register the range in the tree. */ vm_phys_fictitious_init_range(fp, start, page_count, memattr); return (0); } else if (pe > first_page && (pe - first_page) < vm_page_array_size) { /* * We have a segment that ends inside of vm_page_array, * but starts outside of it. */ fp = &vm_page_array[0]; dpage_count = pe - first_page; vm_phys_fictitious_init_range(fp, ptoa(first_page), dpage_count, memattr); end -= ptoa(dpage_count); page_count -= dpage_count; goto alloc; } else if (pi < first_page && pe > (first_page + vm_page_array_size)) { /* * Trying to register a fictitious range that expands before * and after vm_page_array. */ return (EINVAL); } else { alloc: #endif fp = malloc(page_count * sizeof(struct vm_page), M_FICT_PAGES, M_WAITOK); #ifdef VM_PHYSSEG_DENSE } #endif vm_phys_fictitious_init_range(fp, start, page_count, memattr); seg = malloc(sizeof(*seg), M_FICT_PAGES, M_WAITOK | M_ZERO); seg->start = start; seg->end = end; seg->first_page = fp; rw_wlock(&vm_phys_fictitious_reg_lock); RB_INSERT(fict_tree, &vm_phys_fictitious_tree, seg); rw_wunlock(&vm_phys_fictitious_reg_lock); return (0); } void vm_phys_fictitious_unreg_range(vm_paddr_t start, vm_paddr_t end) { struct vm_phys_fictitious_seg *seg, tmp; #ifdef VM_PHYSSEG_DENSE long pi, pe; #endif KASSERT(start < end, ("Start of segment isn't less than end (start: %jx end: %jx)", (uintmax_t)start, (uintmax_t)end)); #ifdef VM_PHYSSEG_DENSE pi = atop(start); pe = atop(end); if (pi >= first_page && (pi - first_page) < vm_page_array_size) { if ((pe - first_page) <= vm_page_array_size) { /* * This segment was allocated using vm_page_array * only, there's nothing to do since those pages * were never added to the tree. */ return; } /* * We have a segment that starts inside * of vm_page_array, but ends outside of it. * * Calculate how many pages were added to the * tree and free them. */ start = ptoa(first_page + vm_page_array_size); } else if (pe > first_page && (pe - first_page) < vm_page_array_size) { /* * We have a segment that ends inside of vm_page_array, * but starts outside of it. */ end = ptoa(first_page); } else if (pi < first_page && pe > (first_page + vm_page_array_size)) { /* Since it's not possible to register such a range, panic. */ panic( "Unregistering not registered fictitious range [%#jx:%#jx]", (uintmax_t)start, (uintmax_t)end); } #endif tmp.start = start; tmp.end = 0; rw_wlock(&vm_phys_fictitious_reg_lock); seg = RB_FIND(fict_tree, &vm_phys_fictitious_tree, &tmp); if (seg->start != start || seg->end != end) { rw_wunlock(&vm_phys_fictitious_reg_lock); panic( "Unregistering not registered fictitious range [%#jx:%#jx]", (uintmax_t)start, (uintmax_t)end); } RB_REMOVE(fict_tree, &vm_phys_fictitious_tree, seg); rw_wunlock(&vm_phys_fictitious_reg_lock); free(seg->first_page, M_FICT_PAGES); free(seg, M_FICT_PAGES); } /* * Free a contiguous, power of two-sized set of physical pages. * * The free page queues must be locked. */ void vm_phys_free_pages(vm_page_t m, int order) { struct vm_freelist *fl; struct vm_phys_seg *seg; vm_paddr_t pa; vm_page_t m_buddy; KASSERT(m->order == VM_NFREEORDER, ("vm_phys_free_pages: page %p has unexpected order %d", m, m->order)); KASSERT(m->pool < VM_NFREEPOOL, ("vm_phys_free_pages: page %p has unexpected pool %d", m, m->pool)); KASSERT(order < VM_NFREEORDER, ("vm_phys_free_pages: order %d is out of range", order)); seg = &vm_phys_segs[m->segind]; vm_domain_free_assert_locked(VM_DOMAIN(seg->domain)); if (order < VM_NFREEORDER - 1) { pa = VM_PAGE_TO_PHYS(m); do { pa ^= ((vm_paddr_t)1 << (PAGE_SHIFT + order)); if (pa < seg->start || pa >= seg->end) break; m_buddy = &seg->first_page[atop(pa - seg->start)]; if (m_buddy->order != order) break; fl = (*seg->free_queues)[m_buddy->pool]; vm_freelist_rem(fl, m_buddy, order); if (m_buddy->pool != m->pool) vm_phys_set_pool(m->pool, m_buddy, order); order++; pa &= ~(((vm_paddr_t)1 << (PAGE_SHIFT + order)) - 1); m = &seg->first_page[atop(pa - seg->start)]; } while (order < VM_NFREEORDER - 1); } fl = (*seg->free_queues)[m->pool]; vm_freelist_add(fl, m, order, 1); } /* * Free a contiguous, arbitrarily sized set of physical pages, without * merging across set boundaries. * * The free page queues must be locked. */ void vm_phys_enqueue_contig(vm_page_t m, u_long npages) { struct vm_freelist *fl; struct vm_phys_seg *seg; vm_page_t m_end; vm_paddr_t diff, lo; int order; /* * Avoid unnecessary coalescing by freeing the pages in the largest * possible power-of-two-sized subsets. */ vm_domain_free_assert_locked(vm_pagequeue_domain(m)); seg = &vm_phys_segs[m->segind]; fl = (*seg->free_queues)[m->pool]; m_end = m + npages; /* Free blocks of increasing size. */ lo = atop(VM_PAGE_TO_PHYS(m)); if (m < m_end && (diff = lo ^ (lo + npages - 1)) != 0) { order = min(flsll(diff) - 1, VM_NFREEORDER - 1); m = vm_phys_enq_range(m, roundup2(lo, 1 << order) - lo, fl, 1); } /* Free blocks of maximum size. */ order = VM_NFREEORDER - 1; while (m + (1 << order) <= m_end) { KASSERT(seg == &vm_phys_segs[m->segind], ("%s: page range [%p,%p) spans multiple segments", __func__, m_end - npages, m)); vm_freelist_add(fl, m, order, 1); m += 1 << order; } /* Free blocks of diminishing size. */ vm_phys_enq_beg(m, m_end - m, fl, 1); } /* * Free a contiguous, arbitrarily sized set of physical pages. * * The free page queues must be locked. */ void vm_phys_free_contig(vm_page_t m, u_long npages) { vm_paddr_t lo; vm_page_t m_start, m_end; unsigned max_order, order_start, order_end; vm_domain_free_assert_locked(vm_pagequeue_domain(m)); lo = atop(VM_PAGE_TO_PHYS(m)); max_order = min(flsll(lo ^ (lo + npages)) - 1, VM_NFREEORDER - 1); m_start = m; order_start = ffsll(lo) - 1; if (order_start < max_order) m_start += 1 << order_start; m_end = m + npages; order_end = ffsll(lo + npages) - 1; if (order_end < max_order) m_end -= 1 << order_end; /* * Avoid unnecessary coalescing by freeing the pages at the start and * end of the range last. */ if (m_start < m_end) vm_phys_enqueue_contig(m_start, m_end - m_start); if (order_start < max_order) vm_phys_free_pages(m, order_start); if (order_end < max_order) vm_phys_free_pages(m_end, order_end); } /* * Identify the first address range within segment segind or greater * that matches the domain, lies within the low/high range, and has * enough pages. Return -1 if there is none. */ int vm_phys_find_range(vm_page_t bounds[], int segind, int domain, u_long npages, vm_paddr_t low, vm_paddr_t high) { vm_paddr_t pa_end, pa_start; struct vm_phys_seg *end_seg, *seg; KASSERT(npages > 0, ("npages is zero")); KASSERT(domain >= 0 && domain < vm_ndomains, ("domain out of range")); end_seg = &vm_phys_segs[vm_phys_nsegs]; for (seg = &vm_phys_segs[segind]; seg < end_seg; seg++) { if (seg->domain != domain) continue; if (seg->start >= high) return (-1); pa_start = MAX(low, seg->start); pa_end = MIN(high, seg->end); if (pa_end - pa_start < ptoa(npages)) continue; bounds[0] = &seg->first_page[atop(pa_start - seg->start)]; bounds[1] = &seg->first_page[atop(pa_end - seg->start)]; return (seg - vm_phys_segs); } return (-1); } /* * Search for the given physical page "m" in the free lists. If the search * succeeds, remove "m" from the free lists and return true. Otherwise, return * false, indicating that "m" is not in the free lists. * * The free page queues must be locked. */ bool vm_phys_unfree_page(vm_page_t m) { struct vm_freelist *fl; struct vm_phys_seg *seg; vm_paddr_t pa, pa_half; vm_page_t m_set, m_tmp; int order; /* * First, find the contiguous, power of two-sized set of free * physical pages containing the given physical page "m" and * assign it to "m_set". */ seg = &vm_phys_segs[m->segind]; vm_domain_free_assert_locked(VM_DOMAIN(seg->domain)); for (m_set = m, order = 0; m_set->order == VM_NFREEORDER && order < VM_NFREEORDER - 1; ) { order++; pa = m->phys_addr & (~(vm_paddr_t)0 << (PAGE_SHIFT + order)); if (pa >= seg->start) m_set = &seg->first_page[atop(pa - seg->start)]; else return (false); } if (m_set->order < order) return (false); if (m_set->order == VM_NFREEORDER) return (false); KASSERT(m_set->order < VM_NFREEORDER, ("vm_phys_unfree_page: page %p has unexpected order %d", m_set, m_set->order)); /* * Next, remove "m_set" from the free lists. Finally, extract * "m" from "m_set" using an iterative algorithm: While "m_set" * is larger than a page, shrink "m_set" by returning the half * of "m_set" that does not contain "m" to the free lists. */ fl = (*seg->free_queues)[m_set->pool]; order = m_set->order; vm_freelist_rem(fl, m_set, order); while (order > 0) { order--; pa_half = m_set->phys_addr ^ (1 << (PAGE_SHIFT + order)); if (m->phys_addr < pa_half) m_tmp = &seg->first_page[atop(pa_half - seg->start)]; else { m_tmp = m_set; m_set = &seg->first_page[atop(pa_half - seg->start)]; } vm_freelist_add(fl, m_tmp, order, 0); } KASSERT(m_set == m, ("vm_phys_unfree_page: fatal inconsistency")); return (true); } /* * Find a run of contiguous physical pages, meeting alignment requirements, from * a list of max-sized page blocks, where we need at least two consecutive * blocks to satisfy the (large) page request. */ static vm_page_t vm_phys_find_freelist_contig(struct vm_freelist *fl, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary) { struct vm_phys_seg *seg; vm_page_t m, m_iter, m_ret; vm_paddr_t max_size, size; int max_order; max_order = VM_NFREEORDER - 1; size = npages << PAGE_SHIFT; max_size = (vm_paddr_t)1 << (PAGE_SHIFT + max_order); KASSERT(size > max_size, ("size is too small")); /* * In order to avoid examining any free max-sized page block more than * twice, identify the ones that are first in a physically-contiguous * sequence of such blocks, and only for those walk the sequence to * check if there are enough free blocks starting at a properly aligned * block. Thus, no block is checked for free-ness more than twice. */ TAILQ_FOREACH(m, &fl[max_order].pl, listq) { /* * Skip m unless it is first in a sequence of free max page * blocks >= low in its segment. */ seg = &vm_phys_segs[m->segind]; if (VM_PAGE_TO_PHYS(m) < MAX(low, seg->start)) continue; if (VM_PAGE_TO_PHYS(m) >= max_size && VM_PAGE_TO_PHYS(m) - max_size >= MAX(low, seg->start) && max_order == m[-1 << max_order].order) continue; /* * Advance m_ret from m to the first of the sequence, if any, * that satisfies alignment conditions and might leave enough * space. */ m_ret = m; while (!vm_addr_ok(VM_PAGE_TO_PHYS(m_ret), size, alignment, boundary) && VM_PAGE_TO_PHYS(m_ret) + size <= MIN(high, seg->end) && max_order == m_ret[1 << max_order].order) m_ret += 1 << max_order; /* * Skip m unless some block m_ret in the sequence is properly * aligned, and begins a sequence of enough pages less than * high, and in the same segment. */ if (VM_PAGE_TO_PHYS(m_ret) + size > MIN(high, seg->end)) continue; /* * Skip m unless the blocks to allocate starting at m_ret are * all free. */ for (m_iter = m_ret; m_iter < m_ret + npages && max_order == m_iter->order; m_iter += 1 << max_order) { } if (m_iter < m_ret + npages) continue; return (m_ret); } return (NULL); } /* * Find a run of contiguous physical pages from the specified free list * table. */ static vm_page_t vm_phys_find_queues_contig( struct vm_freelist (*queues)[VM_NFREEPOOL][VM_NFREEORDER_MAX], u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary) { struct vm_freelist *fl; vm_page_t m_ret; vm_paddr_t pa, pa_end, size; int oind, order, pind; KASSERT(npages > 0, ("npages is 0")); KASSERT(powerof2(alignment), ("alignment is not a power of 2")); KASSERT(powerof2(boundary), ("boundary is not a power of 2")); /* Compute the queue that is the best fit for npages. */ order = flsl(npages - 1); /* Search for a large enough free block. */ size = npages << PAGE_SHIFT; for (oind = order; oind < VM_NFREEORDER; oind++) { for (pind = 0; pind < VM_NFREEPOOL; pind++) { fl = (*queues)[pind]; TAILQ_FOREACH(m_ret, &fl[oind].pl, listq) { /* * Determine if the address range starting at pa * is within the given range, satisfies the * given alignment, and does not cross the given * boundary. */ pa = VM_PAGE_TO_PHYS(m_ret); pa_end = pa + size; if (low <= pa && pa_end <= high && vm_addr_ok(pa, size, alignment, boundary)) return (m_ret); } } } if (order < VM_NFREEORDER) return (NULL); /* Search for a long-enough sequence of max-order blocks. */ for (pind = 0; pind < VM_NFREEPOOL; pind++) { fl = (*queues)[pind]; m_ret = vm_phys_find_freelist_contig(fl, npages, low, high, alignment, boundary); if (m_ret != NULL) return (m_ret); } return (NULL); } /* * Allocate a contiguous set of physical pages of the given size * "npages" from the free lists. All of the physical pages must be at * or above the given physical address "low" and below the given * physical address "high". The given value "alignment" determines the * alignment of the first physical page in the set. If the given value * "boundary" is non-zero, then the set of physical pages cannot cross * any physical address boundary that is a multiple of that value. Both * "alignment" and "boundary" must be a power of two. */ vm_page_t vm_phys_alloc_contig(int domain, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary) { vm_paddr_t pa_end, pa_start; struct vm_freelist *fl; vm_page_t m, m_run; struct vm_phys_seg *seg; struct vm_freelist (*queues)[VM_NFREEPOOL][VM_NFREEORDER_MAX]; int oind, segind; KASSERT(npages > 0, ("npages is 0")); KASSERT(powerof2(alignment), ("alignment is not a power of 2")); KASSERT(powerof2(boundary), ("boundary is not a power of 2")); vm_domain_free_assert_locked(VM_DOMAIN(domain)); if (low >= high) return (NULL); queues = NULL; m_run = NULL; for (segind = vm_phys_nsegs - 1; segind >= 0; segind--) { seg = &vm_phys_segs[segind]; if (seg->start >= high || seg->domain != domain) continue; if (low >= seg->end) break; if (low <= seg->start) pa_start = seg->start; else pa_start = low; if (high < seg->end) pa_end = high; else pa_end = seg->end; if (pa_end - pa_start < ptoa(npages)) continue; /* * If a previous segment led to a search using * the same free lists as would this segment, then * we've actually already searched within this * too. So skip it. */ if (seg->free_queues == queues) continue; queues = seg->free_queues; m_run = vm_phys_find_queues_contig(queues, npages, low, high, alignment, boundary); if (m_run != NULL) break; } if (m_run == NULL) return (NULL); /* Allocate pages from the page-range found. */ for (m = m_run; m < &m_run[npages]; m = &m[1 << oind]) { fl = (*queues)[m->pool]; oind = m->order; vm_freelist_rem(fl, m, oind); if (m->pool != VM_FREEPOOL_DEFAULT) vm_phys_set_pool(VM_FREEPOOL_DEFAULT, m, oind); } /* Return excess pages to the free lists. */ fl = (*queues)[VM_FREEPOOL_DEFAULT]; vm_phys_enq_range(&m_run[npages], m - &m_run[npages], fl, 0); /* Return page verified to satisfy conditions of request. */ pa_start = VM_PAGE_TO_PHYS(m_run); KASSERT(low <= pa_start, ("memory allocated below minimum requested range")); KASSERT(pa_start + ptoa(npages) <= high, ("memory allocated above maximum requested range")); seg = &vm_phys_segs[m_run->segind]; KASSERT(seg->domain == domain, ("memory not allocated from specified domain")); KASSERT(vm_addr_ok(pa_start, ptoa(npages), alignment, boundary), ("memory alignment/boundary constraints not satisfied")); return (m_run); } /* * Return the index of the first unused slot which may be the terminating * entry. */ static int vm_phys_avail_count(void) { int i; for (i = 0; phys_avail[i + 1]; i += 2) continue; if (i > PHYS_AVAIL_ENTRIES) panic("Improperly terminated phys_avail %d entries", i); return (i); } /* * Assert that a phys_avail entry is valid. */ static void vm_phys_avail_check(int i) { if (phys_avail[i] & PAGE_MASK) panic("Unaligned phys_avail[%d]: %#jx", i, (intmax_t)phys_avail[i]); if (phys_avail[i+1] & PAGE_MASK) panic("Unaligned phys_avail[%d + 1]: %#jx", i, (intmax_t)phys_avail[i]); if (phys_avail[i + 1] < phys_avail[i]) panic("phys_avail[%d] start %#jx < end %#jx", i, (intmax_t)phys_avail[i], (intmax_t)phys_avail[i+1]); } /* * Return the index of an overlapping phys_avail entry or -1. */ #ifdef NUMA static int vm_phys_avail_find(vm_paddr_t pa) { int i; for (i = 0; phys_avail[i + 1]; i += 2) if (phys_avail[i] <= pa && phys_avail[i + 1] > pa) return (i); return (-1); } #endif /* * Return the index of the largest entry. */ int vm_phys_avail_largest(void) { vm_paddr_t sz, largesz; int largest; int i; largest = 0; largesz = 0; for (i = 0; phys_avail[i + 1]; i += 2) { sz = vm_phys_avail_size(i); if (sz > largesz) { largesz = sz; largest = i; } } return (largest); } vm_paddr_t vm_phys_avail_size(int i) { return (phys_avail[i + 1] - phys_avail[i]); } /* * Split an entry at the address 'pa'. Return zero on success or errno. */ static int vm_phys_avail_split(vm_paddr_t pa, int i) { int cnt; vm_phys_avail_check(i); if (pa <= phys_avail[i] || pa >= phys_avail[i + 1]) panic("vm_phys_avail_split: invalid address"); cnt = vm_phys_avail_count(); if (cnt >= PHYS_AVAIL_ENTRIES) return (ENOSPC); memmove(&phys_avail[i + 2], &phys_avail[i], (cnt - i) * sizeof(phys_avail[0])); phys_avail[i + 1] = pa; phys_avail[i + 2] = pa; vm_phys_avail_check(i); vm_phys_avail_check(i+2); return (0); } /* * Check if a given physical address can be included as part of a crash dump. */ bool vm_phys_is_dumpable(vm_paddr_t pa) { vm_page_t m; int i; if ((m = vm_phys_paddr_to_vm_page(pa)) != NULL) return ((m->flags & PG_NODUMP) == 0); for (i = 0; dump_avail[i] != 0 || dump_avail[i + 1] != 0; i += 2) { if (pa >= dump_avail[i] && pa < dump_avail[i + 1]) return (true); } return (false); } void vm_phys_early_add_seg(vm_paddr_t start, vm_paddr_t end) { struct vm_phys_seg *seg; if (vm_phys_early_nsegs == -1) panic("%s: called after initialization", __func__); if (vm_phys_early_nsegs == nitems(vm_phys_early_segs)) panic("%s: ran out of early segments", __func__); seg = &vm_phys_early_segs[vm_phys_early_nsegs++]; seg->start = start; seg->end = end; } /* * This routine allocates NUMA node specific memory before the page * allocator is bootstrapped. */ vm_paddr_t vm_phys_early_alloc(int domain, size_t alloc_size) { #ifdef NUMA int mem_index; #endif int i, biggestone; vm_paddr_t pa, mem_start, mem_end, size, biggestsize, align; KASSERT(domain == -1 || (domain >= 0 && domain < vm_ndomains), ("%s: invalid domain index %d", __func__, domain)); /* * Search the mem_affinity array for the biggest address * range in the desired domain. This is used to constrain * the phys_avail selection below. */ biggestsize = 0; mem_start = 0; mem_end = -1; #ifdef NUMA mem_index = 0; if (mem_affinity != NULL) { for (i = 0;; i++) { size = mem_affinity[i].end - mem_affinity[i].start; if (size == 0) break; if (domain != -1 && mem_affinity[i].domain != domain) continue; if (size > biggestsize) { mem_index = i; biggestsize = size; } } mem_start = mem_affinity[mem_index].start; mem_end = mem_affinity[mem_index].end; } #endif /* * Now find biggest physical segment in within the desired * numa domain. */ biggestsize = 0; biggestone = 0; for (i = 0; phys_avail[i + 1] != 0; i += 2) { /* skip regions that are out of range */ if (phys_avail[i+1] - alloc_size < mem_start || phys_avail[i+1] > mem_end) continue; size = vm_phys_avail_size(i); if (size > biggestsize) { biggestone = i; biggestsize = size; } } alloc_size = round_page(alloc_size); /* * Grab single pages from the front to reduce fragmentation. */ if (alloc_size == PAGE_SIZE) { pa = phys_avail[biggestone]; phys_avail[biggestone] += PAGE_SIZE; vm_phys_avail_check(biggestone); return (pa); } /* * Naturally align large allocations. */ align = phys_avail[biggestone + 1] & (alloc_size - 1); if (alloc_size + align > biggestsize) panic("cannot find a large enough size\n"); if (align != 0 && vm_phys_avail_split(phys_avail[biggestone + 1] - align, biggestone) != 0) /* Wasting memory. */ phys_avail[biggestone + 1] -= align; phys_avail[biggestone + 1] -= alloc_size; vm_phys_avail_check(biggestone); pa = phys_avail[biggestone + 1]; return (pa); } void vm_phys_early_startup(void) { struct vm_phys_seg *seg; int i; for (i = 0; phys_avail[i + 1] != 0; i += 2) { phys_avail[i] = round_page(phys_avail[i]); phys_avail[i + 1] = trunc_page(phys_avail[i + 1]); } for (i = 0; i < vm_phys_early_nsegs; i++) { seg = &vm_phys_early_segs[i]; vm_phys_add_seg(seg->start, seg->end); } vm_phys_early_nsegs = -1; #ifdef NUMA /* Force phys_avail to be split by domain. */ if (mem_affinity != NULL) { int idx; for (i = 0; mem_affinity[i].end != 0; i++) { idx = vm_phys_avail_find(mem_affinity[i].start); if (idx != -1 && phys_avail[idx] != mem_affinity[i].start) vm_phys_avail_split(mem_affinity[i].start, idx); idx = vm_phys_avail_find(mem_affinity[i].end); if (idx != -1 && phys_avail[idx] != mem_affinity[i].end) vm_phys_avail_split(mem_affinity[i].end, idx); } } #endif } #ifdef DDB /* * Show the number of physical pages in each of the free lists. */ DB_SHOW_COMMAND_FLAGS(freepages, db_show_freepages, DB_CMD_MEMSAFE) { struct vm_freelist *fl; int flind, oind, pind, dom; for (dom = 0; dom < vm_ndomains; dom++) { db_printf("DOMAIN: %d\n", dom); for (flind = 0; flind < vm_nfreelists; flind++) { db_printf("FREE LIST %d:\n" "\n ORDER (SIZE) | NUMBER" "\n ", flind); for (pind = 0; pind < VM_NFREEPOOL; pind++) db_printf(" | POOL %d", pind); db_printf("\n-- "); for (pind = 0; pind < VM_NFREEPOOL; pind++) db_printf("-- -- "); db_printf("--\n"); for (oind = VM_NFREEORDER - 1; oind >= 0; oind--) { db_printf(" %2.2d (%6.6dK)", oind, 1 << (PAGE_SHIFT - 10 + oind)); for (pind = 0; pind < VM_NFREEPOOL; pind++) { fl = vm_phys_free_queues[dom][flind][pind]; db_printf(" | %6.6d", fl[oind].lcnt); } db_printf("\n"); } db_printf("\n"); } db_printf("\n"); } } #endif diff --git a/sys/vm/vm_phys.h b/sys/vm/vm_phys.h index f3a41785b1b5..1f502ad745c4 100644 --- a/sys/vm/vm_phys.h +++ b/sys/vm/vm_phys.h @@ -1,152 +1,152 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2002-2006 Rice University * Copyright (c) 2007 Alan L. Cox * All rights reserved. * * This software was developed for the FreeBSD Project by Alan L. Cox, * Olivier Crameri, Peter Druschel, Sitaram Iyer, and Juan Navarro. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Physical memory system definitions */ #ifndef _VM_PHYS_H_ #define _VM_PHYS_H_ #ifdef _KERNEL #include extern vm_paddr_t phys_avail[]; /* Domains must be dense (non-sparse) and zero-based. */ struct mem_affinity { vm_paddr_t start; vm_paddr_t end; int domain; }; #ifdef NUMA extern struct mem_affinity *mem_affinity; extern int *mem_locality; #endif /* * The following functions are only to be used by the virtual memory system. */ void vm_phys_add_seg(vm_paddr_t start, vm_paddr_t end); vm_page_t vm_phys_alloc_contig(int domain, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary); vm_page_t vm_phys_alloc_freelist_pages(int domain, int freelist, int pool, int order); int vm_phys_alloc_npages(int domain, int pool, int npages, vm_page_t ma[]); vm_page_t vm_phys_alloc_pages(int domain, int pool, int order); int vm_phys_domain_match(int prefer, vm_paddr_t low, vm_paddr_t high); void vm_phys_enqueue_contig(vm_page_t m, u_long npages); int vm_phys_fictitious_reg_range(vm_paddr_t start, vm_paddr_t end, vm_memattr_t memattr); void vm_phys_fictitious_unreg_range(vm_paddr_t start, vm_paddr_t end); vm_page_t vm_phys_fictitious_to_vm_page(vm_paddr_t pa); int vm_phys_find_range(vm_page_t bounds[], int segind, int domain, u_long npages, vm_paddr_t low, vm_paddr_t high); void vm_phys_free_contig(vm_page_t m, u_long npages); void vm_phys_free_pages(vm_page_t m, int order); void vm_phys_init(void); vm_page_t vm_phys_paddr_to_vm_page(vm_paddr_t pa); void vm_phys_register_domains(int ndomains, struct mem_affinity *affinity, int *locality); bool vm_phys_unfree_page(vm_page_t m); int vm_phys_mem_affinity(int f, int t); void vm_phys_early_add_seg(vm_paddr_t start, vm_paddr_t end); vm_paddr_t vm_phys_early_alloc(int domain, size_t alloc_size); void vm_phys_early_startup(void); int vm_phys_avail_largest(void); vm_paddr_t vm_phys_avail_size(int i); bool vm_phys_is_dumpable(vm_paddr_t pa); static inline int -vm_phys_domain(vm_paddr_t pa) +vm_phys_domain(vm_paddr_t pa __numa_used) { #ifdef NUMA int i; if (vm_ndomains == 1) return (0); for (i = 0; mem_affinity[i].end != 0; i++) if (mem_affinity[i].start <= pa && mem_affinity[i].end >= pa) return (mem_affinity[i].domain); return (-1); #else return (0); #endif } /* * Find the segind for the first segment at or after the given physical address. */ static inline int vm_phys_lookup_segind(vm_paddr_t pa) { u_int hi, lo, mid; lo = 0; hi = vm_phys_nsegs; while (lo != hi) { /* * for i in [0, lo), segs[i].end <= pa * for i in [hi, nsegs), segs[i].end > pa */ mid = lo + (hi - lo) / 2; if (vm_phys_segs[mid].end <= pa) lo = mid + 1; else hi = mid; } return (lo); } /* * Find the segment corresponding to the given physical address. */ static inline struct vm_phys_seg * vm_phys_paddr_to_seg(vm_paddr_t pa) { struct vm_phys_seg *seg; int segind; segind = vm_phys_lookup_segind(pa); if (segind < vm_phys_nsegs) { seg = &vm_phys_segs[segind]; if (pa >= seg->start) return (seg); } return (NULL); } #endif /* _KERNEL */ #endif /* !_VM_PHYS_H_ */