Index: head/sys/sys/elf_common.h =================================================================== --- head/sys/sys/elf_common.h (revision 354789) +++ head/sys/sys/elf_common.h (revision 354790) @@ -1,1463 +1,1464 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2017, 2018 Dell EMC * Copyright (c) 2000, 2001, 2008, 2011, David E. O'Brien * Copyright (c) 1998 John D. Polstra. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SYS_ELF_COMMON_H_ #define _SYS_ELF_COMMON_H_ 1 /* * ELF definitions that are independent of architecture or word size. */ /* * Note header. The ".note" section contains an array of notes. Each * begins with this header, aligned to a word boundary. Immediately * following the note header is n_namesz bytes of name, padded to the * next word boundary. Then comes n_descsz bytes of descriptor, again * padded to a word boundary. The values of n_namesz and n_descsz do * not include the padding. */ typedef struct { u_int32_t n_namesz; /* Length of name. */ u_int32_t n_descsz; /* Length of descriptor. */ u_int32_t n_type; /* Type of this note. */ } Elf_Note; typedef Elf_Note Elf_Nhdr; /* * Option kinds. */ #define ODK_NULL 0 /* undefined */ #define ODK_REGINFO 1 /* register usage info */ #define ODK_EXCEPTIONS 2 /* exception processing info */ #define ODK_PAD 3 /* section padding */ #define ODK_HWPATCH 4 /* hardware patch applied */ #define ODK_FILL 5 /* fill value used by the linker */ #define ODK_TAGS 6 /* reserved space for tools */ #define ODK_HWAND 7 /* hardware AND patch applied */ #define ODK_HWOR 8 /* hardware OR patch applied */ #define ODK_GP_GROUP 9 /* GP group for text/data sections */ #define ODK_IDENT 10 /* ID information */ #define ODK_PAGESIZE 11 /* page size information */ /* * ODK_EXCEPTIONS info field masks. */ #define OEX_FPU_MIN 0x0000001f /* min FPU exception required */ #define OEX_FPU_MAX 0x00001f00 /* max FPU exception allowed */ #define OEX_PAGE0 0x00010000 /* page zero must be mapped */ #define OEX_SMM 0x00020000 /* run in sequential memory mode */ #define OEX_PRECISEFP 0x00040000 /* run in precise FP exception mode */ #define OEX_DISMISS 0x00080000 /* dismiss invalid address traps */ /* * ODK_PAD info field masks. */ #define OPAD_PREFIX 0x0001 #define OPAD_POSTFIX 0x0002 #define OPAD_SYMBOL 0x0004 /* * ODK_HWPATCH info field masks. */ #define OHW_R4KEOP 0x00000001 /* patch for R4000 branch at end-of-page bug */ #define OHW_R8KPFETCH 0x00000002 /* R8000 prefetch bug may occur */ #define OHW_R5KEOP 0x00000004 /* patch for R5000 branch at end-of-page bug */ #define OHW_R5KCVTL 0x00000008 /* R5000 cvt.[ds].l bug: clean == 1 */ #define OHW_R10KLDL 0x00000010UL /* need patch for R10000 misaligned load */ /* * ODK_HWAND/ODK_HWOR info field and hwp_flags[12] masks. */ #define OHWA0_R4KEOP_CHECKED 0x00000001 /* object checked for R4000 end-of-page bug */ #define OHWA0_R4KEOP_CLEAN 0x00000002 /* object verified clean for R4000 end-of-page bug */ #define OHWO0_FIXADE 0x00000001 /* object requires call to fixade */ /* * ODK_IDENT/ODK_GP_GROUP info field masks. */ #define OGP_GROUP 0x0000ffff /* GP group number */ #define OGP_SELF 0x00010000 /* GP group is self-contained */ /* * The header for GNU-style hash sections. */ typedef struct { u_int32_t gh_nbuckets; /* Number of hash buckets. */ u_int32_t gh_symndx; /* First visible symbol in .dynsym. */ u_int32_t gh_maskwords; /* #maskwords used in bloom filter. */ u_int32_t gh_shift2; /* Bloom filter shift count. */ } Elf_GNU_Hash_Header; /* Indexes into the e_ident array. Keep synced with http://www.sco.com/developers/gabi/latest/ch4.eheader.html */ #define EI_MAG0 0 /* Magic number, byte 0. */ #define EI_MAG1 1 /* Magic number, byte 1. */ #define EI_MAG2 2 /* Magic number, byte 2. */ #define EI_MAG3 3 /* Magic number, byte 3. */ #define EI_CLASS 4 /* Class of machine. */ #define EI_DATA 5 /* Data format. */ #define EI_VERSION 6 /* ELF format version. */ #define EI_OSABI 7 /* Operating system / ABI identification */ #define EI_ABIVERSION 8 /* ABI version */ #define OLD_EI_BRAND 8 /* Start of architecture identification. */ #define EI_PAD 9 /* Start of padding (per SVR4 ABI). */ #define EI_NIDENT 16 /* Size of e_ident array. */ /* Values for the magic number bytes. */ #define ELFMAG0 0x7f #define ELFMAG1 'E' #define ELFMAG2 'L' #define ELFMAG3 'F' #define ELFMAG "\177ELF" /* magic string */ #define SELFMAG 4 /* magic string size */ /* Values for e_ident[EI_VERSION] and e_version. */ #define EV_NONE 0 #define EV_CURRENT 1 /* Values for e_ident[EI_CLASS]. */ #define ELFCLASSNONE 0 /* Unknown class. */ #define ELFCLASS32 1 /* 32-bit architecture. */ #define ELFCLASS64 2 /* 64-bit architecture. */ /* Values for e_ident[EI_DATA]. */ #define ELFDATANONE 0 /* Unknown data format. */ #define ELFDATA2LSB 1 /* 2's complement little-endian. */ #define ELFDATA2MSB 2 /* 2's complement big-endian. */ /* Values for e_ident[EI_OSABI]. */ #define ELFOSABI_NONE 0 /* UNIX System V ABI */ #define ELFOSABI_HPUX 1 /* HP-UX operating system */ #define ELFOSABI_NETBSD 2 /* NetBSD */ #define ELFOSABI_LINUX 3 /* GNU/Linux */ #define ELFOSABI_HURD 4 /* GNU/Hurd */ #define ELFOSABI_86OPEN 5 /* 86Open common IA32 ABI */ #define ELFOSABI_SOLARIS 6 /* Solaris */ #define ELFOSABI_AIX 7 /* AIX */ #define ELFOSABI_IRIX 8 /* IRIX */ #define ELFOSABI_FREEBSD 9 /* FreeBSD */ #define ELFOSABI_TRU64 10 /* TRU64 UNIX */ #define ELFOSABI_MODESTO 11 /* Novell Modesto */ #define ELFOSABI_OPENBSD 12 /* OpenBSD */ #define ELFOSABI_OPENVMS 13 /* Open VMS */ #define ELFOSABI_NSK 14 /* HP Non-Stop Kernel */ #define ELFOSABI_AROS 15 /* Amiga Research OS */ #define ELFOSABI_FENIXOS 16 /* FenixOS */ #define ELFOSABI_CLOUDABI 17 /* Nuxi CloudABI */ #define ELFOSABI_ARM_AEABI 64 /* ARM EABI */ #define ELFOSABI_ARM 97 /* ARM */ #define ELFOSABI_STANDALONE 255 /* Standalone (embedded) application */ #define ELFOSABI_SYSV ELFOSABI_NONE /* symbol used in old spec */ #define ELFOSABI_MONTEREY ELFOSABI_AIX /* Monterey */ #define ELFOSABI_GNU ELFOSABI_LINUX /* e_ident */ #define IS_ELF(ehdr) ((ehdr).e_ident[EI_MAG0] == ELFMAG0 && \ (ehdr).e_ident[EI_MAG1] == ELFMAG1 && \ (ehdr).e_ident[EI_MAG2] == ELFMAG2 && \ (ehdr).e_ident[EI_MAG3] == ELFMAG3) /* Values for e_type. */ #define ET_NONE 0 /* Unknown type. */ #define ET_REL 1 /* Relocatable. */ #define ET_EXEC 2 /* Executable. */ #define ET_DYN 3 /* Shared object. */ #define ET_CORE 4 /* Core file. */ #define ET_LOOS 0xfe00 /* First operating system specific. */ #define ET_HIOS 0xfeff /* Last operating system-specific. */ #define ET_LOPROC 0xff00 /* First processor-specific. */ #define ET_HIPROC 0xffff /* Last processor-specific. */ /* Values for e_machine. */ #define EM_NONE 0 /* Unknown machine. */ #define EM_M32 1 /* AT&T WE32100. */ #define EM_SPARC 2 /* Sun SPARC. */ #define EM_386 3 /* Intel i386. */ #define EM_68K 4 /* Motorola 68000. */ #define EM_88K 5 /* Motorola 88000. */ #define EM_IAMCU 6 /* Intel MCU. */ #define EM_860 7 /* Intel i860. */ #define EM_MIPS 8 /* MIPS R3000 Big-Endian only. */ #define EM_S370 9 /* IBM System/370. */ #define EM_MIPS_RS3_LE 10 /* MIPS R3000 Little-Endian. */ #define EM_PARISC 15 /* HP PA-RISC. */ #define EM_VPP500 17 /* Fujitsu VPP500. */ #define EM_SPARC32PLUS 18 /* SPARC v8plus. */ #define EM_960 19 /* Intel 80960. */ #define EM_PPC 20 /* PowerPC 32-bit. */ #define EM_PPC64 21 /* PowerPC 64-bit. */ #define EM_S390 22 /* IBM System/390. */ #define EM_V800 36 /* NEC V800. */ #define EM_FR20 37 /* Fujitsu FR20. */ #define EM_RH32 38 /* TRW RH-32. */ #define EM_RCE 39 /* Motorola RCE. */ #define EM_ARM 40 /* ARM. */ #define EM_SH 42 /* Hitachi SH. */ #define EM_SPARCV9 43 /* SPARC v9 64-bit. */ #define EM_TRICORE 44 /* Siemens TriCore embedded processor. */ #define EM_ARC 45 /* Argonaut RISC Core. */ #define EM_H8_300 46 /* Hitachi H8/300. */ #define EM_H8_300H 47 /* Hitachi H8/300H. */ #define EM_H8S 48 /* Hitachi H8S. */ #define EM_H8_500 49 /* Hitachi H8/500. */ #define EM_IA_64 50 /* Intel IA-64 Processor. */ #define EM_MIPS_X 51 /* Stanford MIPS-X. */ #define EM_COLDFIRE 52 /* Motorola ColdFire. */ #define EM_68HC12 53 /* Motorola M68HC12. */ #define EM_MMA 54 /* Fujitsu MMA. */ #define EM_PCP 55 /* Siemens PCP. */ #define EM_NCPU 56 /* Sony nCPU. */ #define EM_NDR1 57 /* Denso NDR1 microprocessor. */ #define EM_STARCORE 58 /* Motorola Star*Core processor. */ #define EM_ME16 59 /* Toyota ME16 processor. */ #define EM_ST100 60 /* STMicroelectronics ST100 processor. */ #define EM_TINYJ 61 /* Advanced Logic Corp. TinyJ processor. */ #define EM_X86_64 62 /* Advanced Micro Devices x86-64 */ #define EM_AMD64 EM_X86_64 /* Advanced Micro Devices x86-64 (compat) */ #define EM_PDSP 63 /* Sony DSP Processor. */ #define EM_FX66 66 /* Siemens FX66 microcontroller. */ #define EM_ST9PLUS 67 /* STMicroelectronics ST9+ 8/16 microcontroller. */ #define EM_ST7 68 /* STmicroelectronics ST7 8-bit microcontroller. */ #define EM_68HC16 69 /* Motorola MC68HC16 microcontroller. */ #define EM_68HC11 70 /* Motorola MC68HC11 microcontroller. */ #define EM_68HC08 71 /* Motorola MC68HC08 microcontroller. */ #define EM_68HC05 72 /* Motorola MC68HC05 microcontroller. */ #define EM_SVX 73 /* Silicon Graphics SVx. */ #define EM_ST19 74 /* STMicroelectronics ST19 8-bit mc. */ #define EM_VAX 75 /* Digital VAX. */ #define EM_CRIS 76 /* Axis Communications 32-bit embedded processor. */ #define EM_JAVELIN 77 /* Infineon Technologies 32-bit embedded processor. */ #define EM_FIREPATH 78 /* Element 14 64-bit DSP Processor. */ #define EM_ZSP 79 /* LSI Logic 16-bit DSP Processor. */ #define EM_MMIX 80 /* Donald Knuth's educational 64-bit proc. */ #define EM_HUANY 81 /* Harvard University machine-independent object files. */ #define EM_PRISM 82 /* SiTera Prism. */ #define EM_AVR 83 /* Atmel AVR 8-bit microcontroller. */ #define EM_FR30 84 /* Fujitsu FR30. */ #define EM_D10V 85 /* Mitsubishi D10V. */ #define EM_D30V 86 /* Mitsubishi D30V. */ #define EM_V850 87 /* NEC v850. */ #define EM_M32R 88 /* Mitsubishi M32R. */ #define EM_MN10300 89 /* Matsushita MN10300. */ #define EM_MN10200 90 /* Matsushita MN10200. */ #define EM_PJ 91 /* picoJava. */ #define EM_OPENRISC 92 /* OpenRISC 32-bit embedded processor. */ #define EM_ARC_A5 93 /* ARC Cores Tangent-A5. */ #define EM_XTENSA 94 /* Tensilica Xtensa Architecture. */ #define EM_VIDEOCORE 95 /* Alphamosaic VideoCore processor. */ #define EM_TMM_GPP 96 /* Thompson Multimedia General Purpose Processor. */ #define EM_NS32K 97 /* National Semiconductor 32000 series. */ #define EM_TPC 98 /* Tenor Network TPC processor. */ #define EM_SNP1K 99 /* Trebia SNP 1000 processor. */ #define EM_ST200 100 /* STMicroelectronics ST200 microcontroller. */ #define EM_IP2K 101 /* Ubicom IP2xxx microcontroller family. */ #define EM_MAX 102 /* MAX Processor. */ #define EM_CR 103 /* National Semiconductor CompactRISC microprocessor. */ #define EM_F2MC16 104 /* Fujitsu F2MC16. */ #define EM_MSP430 105 /* Texas Instruments embedded microcontroller msp430. */ #define EM_BLACKFIN 106 /* Analog Devices Blackfin (DSP) processor. */ #define EM_SE_C33 107 /* S1C33 Family of Seiko Epson processors. */ #define EM_SEP 108 /* Sharp embedded microprocessor. */ #define EM_ARCA 109 /* Arca RISC Microprocessor. */ #define EM_UNICORE 110 /* Microprocessor series from PKU-Unity Ltd. and MPRC of Peking University */ #define EM_AARCH64 183 /* AArch64 (64-bit ARM) */ #define EM_RISCV 243 /* RISC-V */ /* Non-standard or deprecated. */ #define EM_486 6 /* Intel i486. */ #define EM_MIPS_RS4_BE 10 /* MIPS R4000 Big-Endian */ #define EM_ALPHA_STD 41 /* Digital Alpha (standard value). */ #define EM_ALPHA 0x9026 /* Alpha (written in the absence of an ABI) */ /** * e_flags */ #define EF_ARM_RELEXEC 0x1 #define EF_ARM_HASENTRY 0x2 #define EF_ARM_SYMSARESORTED 0x4 #define EF_ARM_DYNSYMSUSESEGIDX 0x8 #define EF_ARM_MAPSYMSFIRST 0x10 #define EF_ARM_LE8 0x00400000 #define EF_ARM_BE8 0x00800000 #define EF_ARM_EABIMASK 0xFF000000 #define EF_ARM_EABI_UNKNOWN 0x00000000 #define EF_ARM_EABI_VER1 0x01000000 #define EF_ARM_EABI_VER2 0x02000000 #define EF_ARM_EABI_VER3 0x03000000 #define EF_ARM_EABI_VER4 0x04000000 #define EF_ARM_EABI_VER5 0x05000000 #define EF_ARM_INTERWORK 0x00000004 #define EF_ARM_APCS_26 0x00000008 #define EF_ARM_APCS_FLOAT 0x00000010 #define EF_ARM_PIC 0x00000020 #define EF_ARM_ALIGN8 0x00000040 #define EF_ARM_NEW_ABI 0x00000080 #define EF_ARM_OLD_ABI 0x00000100 #define EF_ARM_ABI_FLOAT_SOFT 0x00000200 #define EF_ARM_SOFT_FLOAT EF_ARM_ABI_FLOAT_SOFT /* Pre-V5 ABI name */ #define EF_ARM_ABI_FLOAT_HARD 0x00000400 #define EF_ARM_VFP_FLOAT EF_ARM_ABI_FLOAT_HARD /* Pre-V5 ABI name */ #define EF_ARM_MAVERICK_FLOAT 0x00000800 #define EF_MIPS_NOREORDER 0x00000001 #define EF_MIPS_PIC 0x00000002 /* Contains PIC code */ #define EF_MIPS_CPIC 0x00000004 /* STD PIC calling sequence */ #define EF_MIPS_UCODE 0x00000010 #define EF_MIPS_ABI2 0x00000020 /* N32 */ #define EF_MIPS_OPTIONS_FIRST 0x00000080 #define EF_MIPS_ABI 0x0000F000 #define EF_MIPS_ABI_O32 0x00001000 #define EF_MIPS_ABI_O64 0x00002000 #define EF_MIPS_ABI_EABI32 0x00003000 #define EF_MIPS_ABI_EABI64 0x00004000 #define EF_MIPS_ARCH_ASE 0x0F000000 /* Architectural extensions */ #define EF_MIPS_ARCH_ASE_MDMX 0x08000000 /* MDMX multimedia extension */ #define EF_MIPS_ARCH_ASE_M16 0x04000000 /* MIPS-16 ISA extensions */ #define EF_MIPS_ARCH 0xF0000000 /* Architecture field */ #define EF_MIPS_ARCH_1 0x00000000 /* -mips1 code */ #define EF_MIPS_ARCH_2 0x10000000 /* -mips2 code */ #define EF_MIPS_ARCH_3 0x20000000 /* -mips3 code */ #define EF_MIPS_ARCH_4 0x30000000 /* -mips4 code */ #define EF_MIPS_ARCH_5 0x40000000 /* -mips5 code */ #define EF_MIPS_ARCH_32 0x50000000 /* -mips32 code */ #define EF_MIPS_ARCH_64 0x60000000 /* -mips64 code */ #define EF_MIPS_ARCH_32R2 0x70000000 /* -mips32r2 code */ #define EF_MIPS_ARCH_64R2 0x80000000 /* -mips64r2 code */ #define EF_PPC_EMB 0x80000000 #define EF_PPC_RELOCATABLE 0x00010000 #define EF_PPC_RELOCATABLE_LIB 0x00008000 #define EF_RISCV_RVC 0x00000001 #define EF_RISCV_FLOAT_ABI_MASK 0x00000006 #define EF_RISCV_FLOAT_ABI_SOFT 0x00000000 #define EF_RISCV_FLOAT_ABI_SINGLE 0x000002 #define EF_RISCV_FLOAT_ABI_DOUBLE 0x000004 #define EF_RISCV_FLOAT_ABI_QUAD 0x00000006 #define EF_RISCV_RVE 0x00000008 #define EF_RISCV_TSO 0x00000010 #define EF_SPARC_EXT_MASK 0x00ffff00 #define EF_SPARC_32PLUS 0x00000100 #define EF_SPARC_SUN_US1 0x00000200 #define EF_SPARC_HAL_R1 0x00000200 #define EF_SPARC_SUN_US3 0x00000800 #define EF_SPARCV9_MM 0x00000003 #define EF_SPARCV9_TSO 0x00000000 #define EF_SPARCV9_PSO 0x00000001 #define EF_SPARCV9_RMO 0x00000002 /* Special section indexes. */ #define SHN_UNDEF 0 /* Undefined, missing, irrelevant. */ #define SHN_LORESERVE 0xff00 /* First of reserved range. */ #define SHN_LOPROC 0xff00 /* First processor-specific. */ #define SHN_HIPROC 0xff1f /* Last processor-specific. */ #define SHN_LOOS 0xff20 /* First operating system-specific. */ #define SHN_FBSD_CACHED SHN_LOOS /* Transient, for sys/kern/link_elf_obj linker only: Cached global in local symtab. */ #define SHN_HIOS 0xff3f /* Last operating system-specific. */ #define SHN_ABS 0xfff1 /* Absolute values. */ #define SHN_COMMON 0xfff2 /* Common data. */ #define SHN_XINDEX 0xffff /* Escape -- index stored elsewhere. */ #define SHN_HIRESERVE 0xffff /* Last of reserved range. */ /* sh_type */ #define SHT_NULL 0 /* inactive */ #define SHT_PROGBITS 1 /* program defined information */ #define SHT_SYMTAB 2 /* symbol table section */ #define SHT_STRTAB 3 /* string table section */ #define SHT_RELA 4 /* relocation section with addends */ #define SHT_HASH 5 /* symbol hash table section */ #define SHT_DYNAMIC 6 /* dynamic section */ #define SHT_NOTE 7 /* note section */ #define SHT_NOBITS 8 /* no space section */ #define SHT_REL 9 /* relocation section - no addends */ #define SHT_SHLIB 10 /* reserved - purpose unknown */ #define SHT_DYNSYM 11 /* dynamic symbol table section */ #define SHT_INIT_ARRAY 14 /* Initialization function pointers. */ #define SHT_FINI_ARRAY 15 /* Termination function pointers. */ #define SHT_PREINIT_ARRAY 16 /* Pre-initialization function ptrs. */ #define SHT_GROUP 17 /* Section group. */ #define SHT_SYMTAB_SHNDX 18 /* Section indexes (see SHN_XINDEX). */ #define SHT_LOOS 0x60000000 /* First of OS specific semantics */ #define SHT_LOSUNW 0x6ffffff4 #define SHT_SUNW_dof 0x6ffffff4 #define SHT_SUNW_cap 0x6ffffff5 #define SHT_GNU_ATTRIBUTES 0x6ffffff5 #define SHT_SUNW_SIGNATURE 0x6ffffff6 #define SHT_GNU_HASH 0x6ffffff6 #define SHT_GNU_LIBLIST 0x6ffffff7 #define SHT_SUNW_ANNOTATE 0x6ffffff7 #define SHT_SUNW_DEBUGSTR 0x6ffffff8 #define SHT_SUNW_DEBUG 0x6ffffff9 #define SHT_SUNW_move 0x6ffffffa #define SHT_SUNW_COMDAT 0x6ffffffb #define SHT_SUNW_syminfo 0x6ffffffc #define SHT_SUNW_verdef 0x6ffffffd #define SHT_GNU_verdef 0x6ffffffd /* Symbol versions provided */ #define SHT_SUNW_verneed 0x6ffffffe #define SHT_GNU_verneed 0x6ffffffe /* Symbol versions required */ #define SHT_SUNW_versym 0x6fffffff #define SHT_GNU_versym 0x6fffffff /* Symbol version table */ #define SHT_HISUNW 0x6fffffff #define SHT_HIOS 0x6fffffff /* Last of OS specific semantics */ #define SHT_LOPROC 0x70000000 /* reserved range for processor */ #define SHT_X86_64_UNWIND 0x70000001 /* unwind information */ #define SHT_AMD64_UNWIND SHT_X86_64_UNWIND #define SHT_ARM_EXIDX 0x70000001 /* Exception index table. */ #define SHT_ARM_PREEMPTMAP 0x70000002 /* BPABI DLL dynamic linking pre-emption map. */ #define SHT_ARM_ATTRIBUTES 0x70000003 /* Object file compatibility attributes. */ #define SHT_ARM_DEBUGOVERLAY 0x70000004 /* See DBGOVL for details. */ #define SHT_ARM_OVERLAYSECTION 0x70000005 /* See DBGOVL for details. */ #define SHT_MIPS_LIBLIST 0x70000000 #define SHT_MIPS_MSYM 0x70000001 #define SHT_MIPS_CONFLICT 0x70000002 #define SHT_MIPS_GPTAB 0x70000003 #define SHT_MIPS_UCODE 0x70000004 #define SHT_MIPS_DEBUG 0x70000005 #define SHT_MIPS_REGINFO 0x70000006 #define SHT_MIPS_PACKAGE 0x70000007 #define SHT_MIPS_PACKSYM 0x70000008 #define SHT_MIPS_RELD 0x70000009 #define SHT_MIPS_IFACE 0x7000000b #define SHT_MIPS_CONTENT 0x7000000c #define SHT_MIPS_OPTIONS 0x7000000d #define SHT_MIPS_DELTASYM 0x7000001b #define SHT_MIPS_DELTAINST 0x7000001c #define SHT_MIPS_DELTACLASS 0x7000001d #define SHT_MIPS_DWARF 0x7000001e /* MIPS gcc uses MIPS_DWARF */ #define SHT_MIPS_DELTADECL 0x7000001f #define SHT_MIPS_SYMBOL_LIB 0x70000020 #define SHT_MIPS_EVENTS 0x70000021 #define SHT_MIPS_TRANSLATE 0x70000022 #define SHT_MIPS_PIXIE 0x70000023 #define SHT_MIPS_XLATE 0x70000024 #define SHT_MIPS_XLATE_DEBUG 0x70000025 #define SHT_MIPS_WHIRL 0x70000026 #define SHT_MIPS_EH_REGION 0x70000027 #define SHT_MIPS_XLATE_OLD 0x70000028 #define SHT_MIPS_PDR_EXCEPTION 0x70000029 #define SHT_MIPS_ABIFLAGS 0x7000002a #define SHT_SPARC_GOTDATA 0x70000000 #define SHTORDERED #define SHT_HIPROC 0x7fffffff /* specific section header types */ #define SHT_LOUSER 0x80000000 /* reserved range for application */ #define SHT_HIUSER 0xffffffff /* specific indexes */ /* Flags for sh_flags. */ #define SHF_WRITE 0x1 /* Section contains writable data. */ #define SHF_ALLOC 0x2 /* Section occupies memory. */ #define SHF_EXECINSTR 0x4 /* Section contains instructions. */ #define SHF_MERGE 0x10 /* Section may be merged. */ #define SHF_STRINGS 0x20 /* Section contains strings. */ #define SHF_INFO_LINK 0x40 /* sh_info holds section index. */ #define SHF_LINK_ORDER 0x80 /* Special ordering requirements. */ #define SHF_OS_NONCONFORMING 0x100 /* OS-specific processing required. */ #define SHF_GROUP 0x200 /* Member of section group. */ #define SHF_TLS 0x400 /* Section contains TLS data. */ #define SHF_COMPRESSED 0x800 /* Section contains compressed data. */ #define SHF_MASKOS 0x0ff00000 /* OS-specific semantics. */ #define SHF_MASKPROC 0xf0000000 /* Processor-specific semantics. */ /* Flags for section groups. */ #define GRP_COMDAT 0x1 /* COMDAT semantics. */ /* * Flags / mask for .gnu.versym sections. */ #define VERSYM_VERSION 0x7fff #define VERSYM_HIDDEN 0x8000 /* Values for p_type. */ #define PT_NULL 0 /* Unused entry. */ #define PT_LOAD 1 /* Loadable segment. */ #define PT_DYNAMIC 2 /* Dynamic linking information segment. */ #define PT_INTERP 3 /* Pathname of interpreter. */ #define PT_NOTE 4 /* Auxiliary information. */ #define PT_SHLIB 5 /* Reserved (not used). */ #define PT_PHDR 6 /* Location of program header itself. */ #define PT_TLS 7 /* Thread local storage segment */ #define PT_LOOS 0x60000000 /* First OS-specific. */ #define PT_SUNW_UNWIND 0x6464e550 /* amd64 UNWIND program header */ #define PT_GNU_EH_FRAME 0x6474e550 #define PT_GNU_STACK 0x6474e551 #define PT_GNU_RELRO 0x6474e552 #define PT_DUMP_DELTA 0x6fb5d000 /* va->pa map for kernel dumps (currently arm). */ #define PT_LOSUNW 0x6ffffffa #define PT_SUNWBSS 0x6ffffffa /* Sun Specific segment */ #define PT_SUNWSTACK 0x6ffffffb /* describes the stack segment */ #define PT_SUNWDTRACE 0x6ffffffc /* private */ #define PT_SUNWCAP 0x6ffffffd /* hard/soft capabilities segment */ #define PT_HISUNW 0x6fffffff #define PT_HIOS 0x6fffffff /* Last OS-specific. */ #define PT_LOPROC 0x70000000 /* First processor-specific type. */ #define PT_ARM_ARCHEXT 0x70000000 /* ARM arch compat information. */ #define PT_ARM_EXIDX 0x70000001 /* ARM exception unwind tables. */ #define PT_HIPROC 0x7fffffff /* Last processor-specific type. */ #define PT_OPENBSD_RANDOMIZE 0x65A3DBE6 /* OpenBSD random data segment */ #define PT_OPENBSD_WXNEEDED 0x65A3DBE7 /* OpenBSD EXEC/WRITE pages needed */ #define PT_OPENBSD_BOOTDATA 0x65A41BE6 /* OpenBSD section for boot args */ /* Values for p_flags. */ #define PF_X 0x1 /* Executable. */ #define PF_W 0x2 /* Writable. */ #define PF_R 0x4 /* Readable. */ #define PF_MASKOS 0x0ff00000 /* Operating system-specific. */ #define PF_MASKPROC 0xf0000000 /* Processor-specific. */ /* Extended program header index. */ #define PN_XNUM 0xffff /* Values for d_tag. */ #define DT_NULL 0 /* Terminating entry. */ #define DT_NEEDED 1 /* String table offset of a needed shared library. */ #define DT_PLTRELSZ 2 /* Total size in bytes of PLT relocations. */ #define DT_PLTGOT 3 /* Processor-dependent address. */ #define DT_HASH 4 /* Address of symbol hash table. */ #define DT_STRTAB 5 /* Address of string table. */ #define DT_SYMTAB 6 /* Address of symbol table. */ #define DT_RELA 7 /* Address of ElfNN_Rela relocations. */ #define DT_RELASZ 8 /* Total size of ElfNN_Rela relocations. */ #define DT_RELAENT 9 /* Size of each ElfNN_Rela relocation entry. */ #define DT_STRSZ 10 /* Size of string table. */ #define DT_SYMENT 11 /* Size of each symbol table entry. */ #define DT_INIT 12 /* Address of initialization function. */ #define DT_FINI 13 /* Address of finalization function. */ #define DT_SONAME 14 /* String table offset of shared object name. */ #define DT_RPATH 15 /* String table offset of library path. [sup] */ #define DT_SYMBOLIC 16 /* Indicates "symbolic" linking. [sup] */ #define DT_REL 17 /* Address of ElfNN_Rel relocations. */ #define DT_RELSZ 18 /* Total size of ElfNN_Rel relocations. */ #define DT_RELENT 19 /* Size of each ElfNN_Rel relocation. */ #define DT_PLTREL 20 /* Type of relocation used for PLT. */ #define DT_DEBUG 21 /* Reserved (not used). */ #define DT_TEXTREL 22 /* Indicates there may be relocations in non-writable segments. [sup] */ #define DT_JMPREL 23 /* Address of PLT relocations. */ #define DT_BIND_NOW 24 /* [sup] */ #define DT_INIT_ARRAY 25 /* Address of the array of pointers to initialization functions */ #define DT_FINI_ARRAY 26 /* Address of the array of pointers to termination functions */ #define DT_INIT_ARRAYSZ 27 /* Size in bytes of the array of initialization functions. */ #define DT_FINI_ARRAYSZ 28 /* Size in bytes of the array of termination functions. */ #define DT_RUNPATH 29 /* String table offset of a null-terminated library search path string. */ #define DT_FLAGS 30 /* Object specific flag values. */ #define DT_ENCODING 32 /* Values greater than or equal to DT_ENCODING and less than DT_LOOS follow the rules for the interpretation of the d_un union as follows: even == 'd_ptr', odd == 'd_val' or none */ #define DT_PREINIT_ARRAY 32 /* Address of the array of pointers to pre-initialization functions. */ #define DT_PREINIT_ARRAYSZ 33 /* Size in bytes of the array of pre-initialization functions. */ #define DT_MAXPOSTAGS 34 /* number of positive tags */ #define DT_LOOS 0x6000000d /* First OS-specific */ #define DT_SUNW_AUXILIARY 0x6000000d /* symbol auxiliary name */ #define DT_SUNW_RTLDINF 0x6000000e /* ld.so.1 info (private) */ #define DT_SUNW_FILTER 0x6000000f /* symbol filter name */ #define DT_SUNW_CAP 0x60000010 /* hardware/software */ #define DT_SUNW_ASLR 0x60000023 /* ASLR control */ #define DT_HIOS 0x6ffff000 /* Last OS-specific */ /* * DT_* entries which fall between DT_VALRNGHI & DT_VALRNGLO use the * Dyn.d_un.d_val field of the Elf*_Dyn structure. */ #define DT_VALRNGLO 0x6ffffd00 #define DT_GNU_PRELINKED 0x6ffffdf5 /* prelinking timestamp */ #define DT_GNU_CONFLICTSZ 0x6ffffdf6 /* size of conflict section */ #define DT_GNU_LIBLISTSZ 0x6ffffdf7 /* size of library list */ #define DT_CHECKSUM 0x6ffffdf8 /* elf checksum */ #define DT_PLTPADSZ 0x6ffffdf9 /* pltpadding size */ #define DT_MOVEENT 0x6ffffdfa /* move table entry size */ #define DT_MOVESZ 0x6ffffdfb /* move table size */ #define DT_FEATURE 0x6ffffdfc /* feature holder */ #define DT_FEATURE_1 DT_FEATURE #define DT_POSFLAG_1 0x6ffffdfd /* flags for DT_* entries, effecting */ /* the following DT_* entry. */ /* See DF_P1_* definitions */ #define DT_SYMINSZ 0x6ffffdfe /* syminfo table size (in bytes) */ #define DT_SYMINENT 0x6ffffdff /* syminfo entry size (in bytes) */ #define DT_VALRNGHI 0x6ffffdff /* * DT_* entries which fall between DT_ADDRRNGHI & DT_ADDRRNGLO use the * Dyn.d_un.d_ptr field of the Elf*_Dyn structure. * * If any adjustment is made to the ELF object after it has been * built, these entries will need to be adjusted. */ #define DT_ADDRRNGLO 0x6ffffe00 #define DT_GNU_HASH 0x6ffffef5 /* GNU-style hash table */ #define DT_TLSDESC_PLT 0x6ffffef6 /* loc. of PLT for tlsdesc resolver */ #define DT_TLSDESC_GOT 0x6ffffef7 /* loc. of GOT for tlsdesc resolver */ #define DT_GNU_CONFLICT 0x6ffffef8 /* address of conflict section */ #define DT_GNU_LIBLIST 0x6ffffef9 /* address of library list */ #define DT_CONFIG 0x6ffffefa /* configuration information */ #define DT_DEPAUDIT 0x6ffffefb /* dependency auditing */ #define DT_AUDIT 0x6ffffefc /* object auditing */ #define DT_PLTPAD 0x6ffffefd /* pltpadding (sparcv9) */ #define DT_MOVETAB 0x6ffffefe /* move table */ #define DT_SYMINFO 0x6ffffeff /* syminfo table */ #define DT_ADDRRNGHI 0x6ffffeff #define DT_VERSYM 0x6ffffff0 /* Address of versym section. */ #define DT_RELACOUNT 0x6ffffff9 /* number of RELATIVE relocations */ #define DT_RELCOUNT 0x6ffffffa /* number of RELATIVE relocations */ #define DT_FLAGS_1 0x6ffffffb /* state flags - see DF_1_* defs */ #define DT_VERDEF 0x6ffffffc /* Address of verdef section. */ #define DT_VERDEFNUM 0x6ffffffd /* Number of elems in verdef section */ #define DT_VERNEED 0x6ffffffe /* Address of verneed section. */ #define DT_VERNEEDNUM 0x6fffffff /* Number of elems in verneed section */ #define DT_LOPROC 0x70000000 /* First processor-specific type. */ #define DT_ARM_SYMTABSZ 0x70000001 #define DT_ARM_PREEMPTMAP 0x70000002 #define DT_SPARC_REGISTER 0x70000001 #define DT_DEPRECATED_SPARC_REGISTER 0x7000001 #define DT_MIPS_RLD_VERSION 0x70000001 #define DT_MIPS_TIME_STAMP 0x70000002 #define DT_MIPS_ICHECKSUM 0x70000003 #define DT_MIPS_IVERSION 0x70000004 #define DT_MIPS_FLAGS 0x70000005 #define DT_MIPS_BASE_ADDRESS 0x70000006 #define DT_MIPS_CONFLICT 0x70000008 #define DT_MIPS_LIBLIST 0x70000009 #define DT_MIPS_LOCAL_GOTNO 0x7000000a #define DT_MIPS_CONFLICTNO 0x7000000b #define DT_MIPS_LIBLISTNO 0x70000010 #define DT_MIPS_SYMTABNO 0x70000011 #define DT_MIPS_UNREFEXTNO 0x70000012 #define DT_MIPS_GOTSYM 0x70000013 #define DT_MIPS_HIPAGENO 0x70000014 #define DT_MIPS_RLD_MAP 0x70000016 #define DT_MIPS_DELTA_CLASS 0x70000017 #define DT_MIPS_DELTA_CLASS_NO 0x70000018 #define DT_MIPS_DELTA_INSTANCE 0x70000019 #define DT_MIPS_DELTA_INSTANCE_NO 0x7000001A #define DT_MIPS_DELTA_RELOC 0x7000001B #define DT_MIPS_DELTA_RELOC_NO 0x7000001C #define DT_MIPS_DELTA_SYM 0x7000001D #define DT_MIPS_DELTA_SYM_NO 0x7000001E #define DT_MIPS_DELTA_CLASSSYM 0x70000020 #define DT_MIPS_DELTA_CLASSSYM_NO 0x70000021 #define DT_MIPS_CXX_FLAGS 0x70000022 #define DT_MIPS_PIXIE_INIT 0x70000023 #define DT_MIPS_SYMBOL_LIB 0x70000024 #define DT_MIPS_LOCALPAGE_GOTIDX 0x70000025 #define DT_MIPS_LOCAL_GOTIDX 0x70000026 #define DT_MIPS_HIDDEN_GOTIDX 0x70000027 #define DT_MIPS_PROTECTED_GOTIDX 0x70000028 #define DT_MIPS_OPTIONS 0x70000029 #define DT_MIPS_INTERFACE 0x7000002A #define DT_MIPS_DYNSTR_ALIGN 0x7000002B #define DT_MIPS_INTERFACE_SIZE 0x7000002C #define DT_MIPS_RLD_TEXT_RESOLVE_ADDR 0x7000002D #define DT_MIPS_PERF_SUFFIX 0x7000002E #define DT_MIPS_COMPACT_SIZE 0x7000002F #define DT_MIPS_GP_VALUE 0x70000030 #define DT_MIPS_AUX_DYNAMIC 0x70000031 #define DT_MIPS_PLTGOT 0x70000032 #define DT_MIPS_RLD_OBJ_UPDATE 0x70000033 #define DT_MIPS_RWPLT 0x70000034 #define DT_MIPS_RLD_MAP_REL 0x70000035 #define DT_PPC_GOT 0x70000000 #define DT_PPC_TLSOPT 0x70000001 #define DT_PPC64_GLINK 0x70000000 #define DT_PPC64_OPD 0x70000001 #define DT_PPC64_OPDSZ 0x70000002 #define DT_PPC64_TLSOPT 0x70000003 #define DT_AUXILIARY 0x7ffffffd /* shared library auxiliary name */ #define DT_USED 0x7ffffffe /* ignored - same as needed */ #define DT_FILTER 0x7fffffff /* shared library filter name */ #define DT_HIPROC 0x7fffffff /* Last processor-specific type. */ /* Values for DT_FLAGS */ #define DF_ORIGIN 0x0001 /* Indicates that the object being loaded may make reference to the $ORIGIN substitution string */ #define DF_SYMBOLIC 0x0002 /* Indicates "symbolic" linking. */ #define DF_TEXTREL 0x0004 /* Indicates there may be relocations in non-writable segments. */ #define DF_BIND_NOW 0x0008 /* Indicates that the dynamic linker should process all relocations for the object containing this entry before transferring control to the program. */ #define DF_STATIC_TLS 0x0010 /* Indicates that the shared object or executable contains code using a static thread-local storage scheme. */ /* Values for DT_FLAGS_1 */ #define DF_1_BIND_NOW 0x00000001 /* Same as DF_BIND_NOW */ #define DF_1_GLOBAL 0x00000002 /* Set the RTLD_GLOBAL for object */ #define DF_1_NODELETE 0x00000008 /* Set the RTLD_NODELETE for object */ #define DF_1_LOADFLTR 0x00000010 /* Immediate loading of filtees */ #define DF_1_NOOPEN 0x00000040 /* Do not allow loading on dlopen() */ #define DF_1_ORIGIN 0x00000080 /* Process $ORIGIN */ #define DF_1_INTERPOSE 0x00000400 /* Interpose all objects but main */ #define DF_1_NODEFLIB 0x00000800 /* Do not search default paths */ /* Values for l_flags. */ #define LL_NONE 0x0 /* no flags */ #define LL_EXACT_MATCH 0x1 /* require an exact match */ #define LL_IGNORE_INT_VER 0x2 /* ignore version incompatibilities */ #define LL_REQUIRE_MINOR 0x4 #define LL_EXPORTS 0x8 #define LL_DELAY_LOAD 0x10 #define LL_DELTA 0x20 /* Note section names */ #define ELF_NOTE_FREEBSD "FreeBSD" #define ELF_NOTE_NETBSD "NetBSD" #define ELF_NOTE_SOLARIS "SUNW Solaris" #define ELF_NOTE_GNU "GNU" /* Values for n_type used in executables. */ #define NT_FREEBSD_ABI_TAG 1 #define NT_FREEBSD_NOINIT_TAG 2 #define NT_FREEBSD_ARCH_TAG 3 #define NT_FREEBSD_FEATURE_CTL 4 /* NT_FREEBSD_FEATURE_CTL desc[0] bits */ #define NT_FREEBSD_FCTL_ASLR_DISABLE 0x00000001 #define NT_FREEBSD_FCTL_PROTMAX_DISABLE 0x00000002 +#define NT_FREEBSD_FCTL_STKGAP_DISABLE 0x00000004 /* Values for n_type. Used in core files. */ #define NT_PRSTATUS 1 /* Process status. */ #define NT_FPREGSET 2 /* Floating point registers. */ #define NT_PRPSINFO 3 /* Process state info. */ #define NT_THRMISC 7 /* Thread miscellaneous info. */ #define NT_PROCSTAT_PROC 8 /* Procstat proc data. */ #define NT_PROCSTAT_FILES 9 /* Procstat files data. */ #define NT_PROCSTAT_VMMAP 10 /* Procstat vmmap data. */ #define NT_PROCSTAT_GROUPS 11 /* Procstat groups data. */ #define NT_PROCSTAT_UMASK 12 /* Procstat umask data. */ #define NT_PROCSTAT_RLIMIT 13 /* Procstat rlimit data. */ #define NT_PROCSTAT_OSREL 14 /* Procstat osreldate data. */ #define NT_PROCSTAT_PSSTRINGS 15 /* Procstat ps_strings data. */ #define NT_PROCSTAT_AUXV 16 /* Procstat auxv data. */ #define NT_PTLWPINFO 17 /* Thread ptrace miscellaneous info. */ #define NT_PPC_VMX 0x100 /* PowerPC Altivec/VMX registers */ #define NT_PPC_VSX 0x102 /* PowerPC VSX registers */ #define NT_X86_XSTATE 0x202 /* x86 XSAVE extended state. */ #define NT_ARM_VFP 0x400 /* ARM VFP registers */ /* GNU note types. */ #define NT_GNU_ABI_TAG 1 #define NT_GNU_HWCAP 2 #define NT_GNU_BUILD_ID 3 #define NT_GNU_GOLD_VERSION 4 #define NT_GNU_PROPERTY_TYPE_0 5 #define GNU_PROPERTY_LOPROC 0xc0000000 #define GNU_PROPERTY_HIPROC 0xdfffffff #define GNU_PROPERTY_X86_FEATURE_1_AND 0xc0000002 #define GNU_PROPERTY_X86_FEATURE_1_IBT 0x00000001 #define GNU_PROPERTY_X86_FEATURE_1_SHSTK 0x00000002 /* Symbol Binding - ELFNN_ST_BIND - st_info */ #define STB_LOCAL 0 /* Local symbol */ #define STB_GLOBAL 1 /* Global symbol */ #define STB_WEAK 2 /* like global - lower precedence */ #define STB_LOOS 10 /* Start of operating system reserved range. */ #define STB_GNU_UNIQUE 10 /* Unique symbol (GNU) */ #define STB_HIOS 12 /* End of operating system reserved range. */ #define STB_LOPROC 13 /* reserved range for processor */ #define STB_HIPROC 15 /* specific semantics. */ /* Symbol type - ELFNN_ST_TYPE - st_info */ #define STT_NOTYPE 0 /* Unspecified type. */ #define STT_OBJECT 1 /* Data object. */ #define STT_FUNC 2 /* Function. */ #define STT_SECTION 3 /* Section. */ #define STT_FILE 4 /* Source file. */ #define STT_COMMON 5 /* Uninitialized common block. */ #define STT_TLS 6 /* TLS object. */ #define STT_NUM 7 #define STT_LOOS 10 /* Reserved range for operating system */ #define STT_GNU_IFUNC 10 #define STT_HIOS 12 /* specific semantics. */ #define STT_LOPROC 13 /* Start of processor reserved range. */ #define STT_SPARC_REGISTER 13 /* SPARC register information. */ #define STT_HIPROC 15 /* End of processor reserved range. */ /* Symbol visibility - ELFNN_ST_VISIBILITY - st_other */ #define STV_DEFAULT 0x0 /* Default visibility (see binding). */ #define STV_INTERNAL 0x1 /* Special meaning in relocatable objects. */ #define STV_HIDDEN 0x2 /* Not visible. */ #define STV_PROTECTED 0x3 /* Visible but not preemptible. */ #define STV_EXPORTED 0x4 #define STV_SINGLETON 0x5 #define STV_ELIMINATE 0x6 /* Special symbol table indexes. */ #define STN_UNDEF 0 /* Undefined symbol index. */ /* Symbol versioning flags. */ #define VER_DEF_CURRENT 1 #define VER_DEF_IDX(x) VER_NDX(x) #define VER_FLG_BASE 0x01 #define VER_FLG_WEAK 0x02 #define VER_NEED_CURRENT 1 #define VER_NEED_WEAK (1u << 15) #define VER_NEED_HIDDEN VER_NDX_HIDDEN #define VER_NEED_IDX(x) VER_NDX(x) #define VER_NDX_LOCAL 0 #define VER_NDX_GLOBAL 1 #define VER_NDX_GIVEN 2 #define VER_NDX_HIDDEN (1u << 15) #define VER_NDX(x) ((x) & ~(1u << 15)) #define CA_SUNW_NULL 0 #define CA_SUNW_HW_1 1 /* first hardware capabilities entry */ #define CA_SUNW_SF_1 2 /* first software capabilities entry */ /* * Syminfo flag values */ #define SYMINFO_FLG_DIRECT 0x0001 /* symbol ref has direct association */ /* to object containing defn. */ #define SYMINFO_FLG_PASSTHRU 0x0002 /* ignored - see SYMINFO_FLG_FILTER */ #define SYMINFO_FLG_COPY 0x0004 /* symbol is a copy-reloc */ #define SYMINFO_FLG_LAZYLOAD 0x0008 /* object containing defn should be */ /* lazily-loaded */ #define SYMINFO_FLG_DIRECTBIND 0x0010 /* ref should be bound directly to */ /* object containing defn. */ #define SYMINFO_FLG_NOEXTDIRECT 0x0020 /* don't let an external reference */ /* directly bind to this symbol */ #define SYMINFO_FLG_FILTER 0x0002 /* symbol ref is associated to a */ #define SYMINFO_FLG_AUXILIARY 0x0040 /* standard or auxiliary filter */ /* * Syminfo.si_boundto values. */ #define SYMINFO_BT_SELF 0xffff /* symbol bound to self */ #define SYMINFO_BT_PARENT 0xfffe /* symbol bound to parent */ #define SYMINFO_BT_NONE 0xfffd /* no special symbol binding */ #define SYMINFO_BT_EXTERN 0xfffc /* symbol defined as external */ #define SYMINFO_BT_LOWRESERVE 0xff00 /* beginning of reserved entries */ /* * Syminfo version values. */ #define SYMINFO_NONE 0 /* Syminfo version */ #define SYMINFO_CURRENT 1 #define SYMINFO_NUM 2 /* Values for ch_type (compressed section headers). */ #define ELFCOMPRESS_ZLIB 1 /* ZLIB/DEFLATE */ #define ELFCOMPRESS_LOOS 0x60000000 /* OS-specific */ #define ELFCOMPRESS_HIOS 0x6fffffff #define ELFCOMPRESS_LOPROC 0x70000000 /* Processor-specific */ #define ELFCOMPRESS_HIPROC 0x7fffffff /* Values for a_type. */ #define AT_NULL 0 /* Terminates the vector. */ #define AT_IGNORE 1 /* Ignored entry. */ #define AT_EXECFD 2 /* File descriptor of program to load. */ #define AT_PHDR 3 /* Program header of program already loaded. */ #define AT_PHENT 4 /* Size of each program header entry. */ #define AT_PHNUM 5 /* Number of program header entries. */ #define AT_PAGESZ 6 /* Page size in bytes. */ #define AT_BASE 7 /* Interpreter's base address. */ #define AT_FLAGS 8 /* Flags. */ #define AT_ENTRY 9 /* Where interpreter should transfer control. */ #define AT_NOTELF 10 /* Program is not ELF ?? */ #define AT_UID 11 /* Real uid. */ #define AT_EUID 12 /* Effective uid. */ #ifndef __powerpc__ #define AT_GID 13 /* Real gid. */ #define AT_EGID 14 /* Effective gid. */ #define AT_EXECPATH 15 /* Path to the executable. */ #define AT_CANARY 16 /* Canary for SSP. */ #define AT_CANARYLEN 17 /* Length of the canary. */ #define AT_OSRELDATE 18 /* OSRELDATE. */ #define AT_NCPUS 19 /* Number of CPUs. */ #define AT_PAGESIZES 20 /* Pagesizes. */ #define AT_PAGESIZESLEN 21 /* Number of pagesizes. */ #else /* defined(__powerpc__) */ #define AT_EXECPATH 13 #define AT_CANARY 14 #define AT_CANARYLEN 15 #define AT_OSRELDATE 16 #define AT_NCPUS 17 #define AT_PAGESIZES 18 #define AT_PAGESIZESLEN 19 #define AT_STACKPROT 21 #endif /* defined(__powerpc__) */ #define AT_TIMEKEEP 22 /* Pointer to timehands. */ #ifndef __powerpc__ #define AT_STACKPROT 23 /* Initial stack protection. */ #endif #define AT_EHDRFLAGS 24 /* e_flags field from elf hdr */ #define AT_HWCAP 25 /* CPU feature flags. */ #define AT_HWCAP2 26 /* CPU feature flags 2. */ #define AT_COUNT 27 /* Count of defined aux entry types. */ /* * Relocation types. * * All machine architectures are defined here to allow tools on one to * handle others. */ #define R_386_NONE 0 /* No relocation. */ #define R_386_32 1 /* Add symbol value. */ #define R_386_PC32 2 /* Add PC-relative symbol value. */ #define R_386_GOT32 3 /* Add PC-relative GOT offset. */ #define R_386_PLT32 4 /* Add PC-relative PLT offset. */ #define R_386_COPY 5 /* Copy data from shared object. */ #define R_386_GLOB_DAT 6 /* Set GOT entry to data address. */ #define R_386_JMP_SLOT 7 /* Set GOT entry to code address. */ #define R_386_RELATIVE 8 /* Add load address of shared object. */ #define R_386_GOTOFF 9 /* Add GOT-relative symbol address. */ #define R_386_GOTPC 10 /* Add PC-relative GOT table address. */ #define R_386_TLS_TPOFF 14 /* Negative offset in static TLS block */ #define R_386_TLS_IE 15 /* Absolute address of GOT for -ve static TLS */ #define R_386_TLS_GOTIE 16 /* GOT entry for negative static TLS block */ #define R_386_TLS_LE 17 /* Negative offset relative to static TLS */ #define R_386_TLS_GD 18 /* 32 bit offset to GOT (index,off) pair */ #define R_386_TLS_LDM 19 /* 32 bit offset to GOT (index,zero) pair */ #define R_386_TLS_GD_32 24 /* 32 bit offset to GOT (index,off) pair */ #define R_386_TLS_GD_PUSH 25 /* pushl instruction for Sun ABI GD sequence */ #define R_386_TLS_GD_CALL 26 /* call instruction for Sun ABI GD sequence */ #define R_386_TLS_GD_POP 27 /* popl instruction for Sun ABI GD sequence */ #define R_386_TLS_LDM_32 28 /* 32 bit offset to GOT (index,zero) pair */ #define R_386_TLS_LDM_PUSH 29 /* pushl instruction for Sun ABI LD sequence */ #define R_386_TLS_LDM_CALL 30 /* call instruction for Sun ABI LD sequence */ #define R_386_TLS_LDM_POP 31 /* popl instruction for Sun ABI LD sequence */ #define R_386_TLS_LDO_32 32 /* 32 bit offset from start of TLS block */ #define R_386_TLS_IE_32 33 /* 32 bit offset to GOT static TLS offset entry */ #define R_386_TLS_LE_32 34 /* 32 bit offset within static TLS block */ #define R_386_TLS_DTPMOD32 35 /* GOT entry containing TLS index */ #define R_386_TLS_DTPOFF32 36 /* GOT entry containing TLS offset */ #define R_386_TLS_TPOFF32 37 /* GOT entry of -ve static TLS offset */ #define R_386_IRELATIVE 42 /* PLT entry resolved indirectly at runtime */ #define R_AARCH64_NONE 0 /* No relocation */ #define R_AARCH64_ABS64 257 /* Absolute offset */ #define R_AARCH64_ABS32 258 /* Absolute, 32-bit overflow check */ #define R_AARCH64_ABS16 259 /* Absolute, 16-bit overflow check */ #define R_AARCH64_PREL64 260 /* PC relative */ #define R_AARCH64_PREL32 261 /* PC relative, 32-bit overflow check */ #define R_AARCH64_PREL16 262 /* PC relative, 16-bit overflow check */ #define R_AARCH64_COPY 1024 /* Copy data from shared object */ #define R_AARCH64_GLOB_DAT 1025 /* Set GOT entry to data address */ #define R_AARCH64_JUMP_SLOT 1026 /* Set GOT entry to code address */ #define R_AARCH64_RELATIVE 1027 /* Add load address of shared object */ #define R_AARCH64_TLS_DTPREL64 1028 #define R_AARCH64_TLS_DTPMOD64 1029 #define R_AARCH64_TLS_TPREL64 1030 #define R_AARCH64_TLSDESC 1031 /* Identify the TLS descriptor */ #define R_AARCH64_IRELATIVE 1032 #define R_ARM_NONE 0 /* No relocation. */ #define R_ARM_PC24 1 #define R_ARM_ABS32 2 #define R_ARM_REL32 3 #define R_ARM_PC13 4 #define R_ARM_ABS16 5 #define R_ARM_ABS12 6 #define R_ARM_THM_ABS5 7 #define R_ARM_ABS8 8 #define R_ARM_SBREL32 9 #define R_ARM_THM_PC22 10 #define R_ARM_THM_PC8 11 #define R_ARM_AMP_VCALL9 12 #define R_ARM_SWI24 13 #define R_ARM_THM_SWI8 14 #define R_ARM_XPC25 15 #define R_ARM_THM_XPC22 16 /* TLS relocations */ #define R_ARM_TLS_DTPMOD32 17 /* ID of module containing symbol */ #define R_ARM_TLS_DTPOFF32 18 /* Offset in TLS block */ #define R_ARM_TLS_TPOFF32 19 /* Offset in static TLS block */ #define R_ARM_COPY 20 /* Copy data from shared object. */ #define R_ARM_GLOB_DAT 21 /* Set GOT entry to data address. */ #define R_ARM_JUMP_SLOT 22 /* Set GOT entry to code address. */ #define R_ARM_RELATIVE 23 /* Add load address of shared object. */ #define R_ARM_GOTOFF 24 /* Add GOT-relative symbol address. */ #define R_ARM_GOTPC 25 /* Add PC-relative GOT table address. */ #define R_ARM_GOT32 26 /* Add PC-relative GOT offset. */ #define R_ARM_PLT32 27 /* Add PC-relative PLT offset. */ #define R_ARM_GNU_VTENTRY 100 #define R_ARM_GNU_VTINHERIT 101 #define R_ARM_RSBREL32 250 #define R_ARM_THM_RPC22 251 #define R_ARM_RREL32 252 #define R_ARM_RABS32 253 #define R_ARM_RPC24 254 #define R_ARM_RBASE 255 /* Name Value Field Calculation */ #define R_IA_64_NONE 0 /* None */ #define R_IA_64_IMM14 0x21 /* immediate14 S + A */ #define R_IA_64_IMM22 0x22 /* immediate22 S + A */ #define R_IA_64_IMM64 0x23 /* immediate64 S + A */ #define R_IA_64_DIR32MSB 0x24 /* word32 MSB S + A */ #define R_IA_64_DIR32LSB 0x25 /* word32 LSB S + A */ #define R_IA_64_DIR64MSB 0x26 /* word64 MSB S + A */ #define R_IA_64_DIR64LSB 0x27 /* word64 LSB S + A */ #define R_IA_64_GPREL22 0x2a /* immediate22 @gprel(S + A) */ #define R_IA_64_GPREL64I 0x2b /* immediate64 @gprel(S + A) */ #define R_IA_64_GPREL32MSB 0x2c /* word32 MSB @gprel(S + A) */ #define R_IA_64_GPREL32LSB 0x2d /* word32 LSB @gprel(S + A) */ #define R_IA_64_GPREL64MSB 0x2e /* word64 MSB @gprel(S + A) */ #define R_IA_64_GPREL64LSB 0x2f /* word64 LSB @gprel(S + A) */ #define R_IA_64_LTOFF22 0x32 /* immediate22 @ltoff(S + A) */ #define R_IA_64_LTOFF64I 0x33 /* immediate64 @ltoff(S + A) */ #define R_IA_64_PLTOFF22 0x3a /* immediate22 @pltoff(S + A) */ #define R_IA_64_PLTOFF64I 0x3b /* immediate64 @pltoff(S + A) */ #define R_IA_64_PLTOFF64MSB 0x3e /* word64 MSB @pltoff(S + A) */ #define R_IA_64_PLTOFF64LSB 0x3f /* word64 LSB @pltoff(S + A) */ #define R_IA_64_FPTR64I 0x43 /* immediate64 @fptr(S + A) */ #define R_IA_64_FPTR32MSB 0x44 /* word32 MSB @fptr(S + A) */ #define R_IA_64_FPTR32LSB 0x45 /* word32 LSB @fptr(S + A) */ #define R_IA_64_FPTR64MSB 0x46 /* word64 MSB @fptr(S + A) */ #define R_IA_64_FPTR64LSB 0x47 /* word64 LSB @fptr(S + A) */ #define R_IA_64_PCREL60B 0x48 /* immediate60 form1 S + A - P */ #define R_IA_64_PCREL21B 0x49 /* immediate21 form1 S + A - P */ #define R_IA_64_PCREL21M 0x4a /* immediate21 form2 S + A - P */ #define R_IA_64_PCREL21F 0x4b /* immediate21 form3 S + A - P */ #define R_IA_64_PCREL32MSB 0x4c /* word32 MSB S + A - P */ #define R_IA_64_PCREL32LSB 0x4d /* word32 LSB S + A - P */ #define R_IA_64_PCREL64MSB 0x4e /* word64 MSB S + A - P */ #define R_IA_64_PCREL64LSB 0x4f /* word64 LSB S + A - P */ #define R_IA_64_LTOFF_FPTR22 0x52 /* immediate22 @ltoff(@fptr(S + A)) */ #define R_IA_64_LTOFF_FPTR64I 0x53 /* immediate64 @ltoff(@fptr(S + A)) */ #define R_IA_64_LTOFF_FPTR32MSB 0x54 /* word32 MSB @ltoff(@fptr(S + A)) */ #define R_IA_64_LTOFF_FPTR32LSB 0x55 /* word32 LSB @ltoff(@fptr(S + A)) */ #define R_IA_64_LTOFF_FPTR64MSB 0x56 /* word64 MSB @ltoff(@fptr(S + A)) */ #define R_IA_64_LTOFF_FPTR64LSB 0x57 /* word64 LSB @ltoff(@fptr(S + A)) */ #define R_IA_64_SEGREL32MSB 0x5c /* word32 MSB @segrel(S + A) */ #define R_IA_64_SEGREL32LSB 0x5d /* word32 LSB @segrel(S + A) */ #define R_IA_64_SEGREL64MSB 0x5e /* word64 MSB @segrel(S + A) */ #define R_IA_64_SEGREL64LSB 0x5f /* word64 LSB @segrel(S + A) */ #define R_IA_64_SECREL32MSB 0x64 /* word32 MSB @secrel(S + A) */ #define R_IA_64_SECREL32LSB 0x65 /* word32 LSB @secrel(S + A) */ #define R_IA_64_SECREL64MSB 0x66 /* word64 MSB @secrel(S + A) */ #define R_IA_64_SECREL64LSB 0x67 /* word64 LSB @secrel(S + A) */ #define R_IA_64_REL32MSB 0x6c /* word32 MSB BD + A */ #define R_IA_64_REL32LSB 0x6d /* word32 LSB BD + A */ #define R_IA_64_REL64MSB 0x6e /* word64 MSB BD + A */ #define R_IA_64_REL64LSB 0x6f /* word64 LSB BD + A */ #define R_IA_64_LTV32MSB 0x74 /* word32 MSB S + A */ #define R_IA_64_LTV32LSB 0x75 /* word32 LSB S + A */ #define R_IA_64_LTV64MSB 0x76 /* word64 MSB S + A */ #define R_IA_64_LTV64LSB 0x77 /* word64 LSB S + A */ #define R_IA_64_PCREL21BI 0x79 /* immediate21 form1 S + A - P */ #define R_IA_64_PCREL22 0x7a /* immediate22 S + A - P */ #define R_IA_64_PCREL64I 0x7b /* immediate64 S + A - P */ #define R_IA_64_IPLTMSB 0x80 /* function descriptor MSB special */ #define R_IA_64_IPLTLSB 0x81 /* function descriptor LSB speciaal */ #define R_IA_64_SUB 0x85 /* immediate64 A - S */ #define R_IA_64_LTOFF22X 0x86 /* immediate22 special */ #define R_IA_64_LDXMOV 0x87 /* immediate22 special */ #define R_IA_64_TPREL14 0x91 /* imm14 @tprel(S + A) */ #define R_IA_64_TPREL22 0x92 /* imm22 @tprel(S + A) */ #define R_IA_64_TPREL64I 0x93 /* imm64 @tprel(S + A) */ #define R_IA_64_TPREL64MSB 0x96 /* word64 MSB @tprel(S + A) */ #define R_IA_64_TPREL64LSB 0x97 /* word64 LSB @tprel(S + A) */ #define R_IA_64_LTOFF_TPREL22 0x9a /* imm22 @ltoff(@tprel(S+A)) */ #define R_IA_64_DTPMOD64MSB 0xa6 /* word64 MSB @dtpmod(S + A) */ #define R_IA_64_DTPMOD64LSB 0xa7 /* word64 LSB @dtpmod(S + A) */ #define R_IA_64_LTOFF_DTPMOD22 0xaa /* imm22 @ltoff(@dtpmod(S+A)) */ #define R_IA_64_DTPREL14 0xb1 /* imm14 @dtprel(S + A) */ #define R_IA_64_DTPREL22 0xb2 /* imm22 @dtprel(S + A) */ #define R_IA_64_DTPREL64I 0xb3 /* imm64 @dtprel(S + A) */ #define R_IA_64_DTPREL32MSB 0xb4 /* word32 MSB @dtprel(S + A) */ #define R_IA_64_DTPREL32LSB 0xb5 /* word32 LSB @dtprel(S + A) */ #define R_IA_64_DTPREL64MSB 0xb6 /* word64 MSB @dtprel(S + A) */ #define R_IA_64_DTPREL64LSB 0xb7 /* word64 LSB @dtprel(S + A) */ #define R_IA_64_LTOFF_DTPREL22 0xba /* imm22 @ltoff(@dtprel(S+A)) */ #define R_MIPS_NONE 0 /* No reloc */ #define R_MIPS_16 1 /* Direct 16 bit */ #define R_MIPS_32 2 /* Direct 32 bit */ #define R_MIPS_REL32 3 /* PC relative 32 bit */ #define R_MIPS_26 4 /* Direct 26 bit shifted */ #define R_MIPS_HI16 5 /* High 16 bit */ #define R_MIPS_LO16 6 /* Low 16 bit */ #define R_MIPS_GPREL16 7 /* GP relative 16 bit */ #define R_MIPS_LITERAL 8 /* 16 bit literal entry */ #define R_MIPS_GOT16 9 /* 16 bit GOT entry */ #define R_MIPS_PC16 10 /* PC relative 16 bit */ #define R_MIPS_CALL16 11 /* 16 bit GOT entry for function */ #define R_MIPS_GPREL32 12 /* GP relative 32 bit */ #define R_MIPS_64 18 /* Direct 64 bit */ #define R_MIPS_GOT_DISP 19 #define R_MIPS_GOT_PAGE 20 #define R_MIPS_GOT_OFST 21 #define R_MIPS_GOT_HI16 22 /* GOT HI 16 bit */ #define R_MIPS_GOT_LO16 23 /* GOT LO 16 bit */ #define R_MIPS_SUB 24 #define R_MIPS_CALLHI16 30 /* upper 16 bit GOT entry for function */ #define R_MIPS_CALLLO16 31 /* lower 16 bit GOT entry for function */ #define R_MIPS_JALR 37 #define R_MIPS_TLS_GD 42 #define R_MIPS_COPY 126 #define R_MIPS_JUMP_SLOT 127 #define R_PPC_NONE 0 /* No relocation. */ #define R_PPC_ADDR32 1 #define R_PPC_ADDR24 2 #define R_PPC_ADDR16 3 #define R_PPC_ADDR16_LO 4 #define R_PPC_ADDR16_HI 5 #define R_PPC_ADDR16_HA 6 #define R_PPC_ADDR14 7 #define R_PPC_ADDR14_BRTAKEN 8 #define R_PPC_ADDR14_BRNTAKEN 9 #define R_PPC_REL24 10 #define R_PPC_REL14 11 #define R_PPC_REL14_BRTAKEN 12 #define R_PPC_REL14_BRNTAKEN 13 #define R_PPC_GOT16 14 #define R_PPC_GOT16_LO 15 #define R_PPC_GOT16_HI 16 #define R_PPC_GOT16_HA 17 #define R_PPC_PLTREL24 18 #define R_PPC_COPY 19 #define R_PPC_GLOB_DAT 20 #define R_PPC_JMP_SLOT 21 #define R_PPC_RELATIVE 22 #define R_PPC_LOCAL24PC 23 #define R_PPC_UADDR32 24 #define R_PPC_UADDR16 25 #define R_PPC_REL32 26 #define R_PPC_PLT32 27 #define R_PPC_PLTREL32 28 #define R_PPC_PLT16_LO 29 #define R_PPC_PLT16_HI 30 #define R_PPC_PLT16_HA 31 #define R_PPC_SDAREL16 32 #define R_PPC_SECTOFF 33 #define R_PPC_SECTOFF_LO 34 #define R_PPC_SECTOFF_HI 35 #define R_PPC_SECTOFF_HA 36 #define R_PPC_IRELATIVE 248 /* * 64-bit relocations */ #define R_PPC64_ADDR64 38 #define R_PPC64_ADDR16_HIGHER 39 #define R_PPC64_ADDR16_HIGHERA 40 #define R_PPC64_ADDR16_HIGHEST 41 #define R_PPC64_ADDR16_HIGHESTA 42 #define R_PPC64_UADDR64 43 #define R_PPC64_REL64 44 #define R_PPC64_PLT64 45 #define R_PPC64_PLTREL64 46 #define R_PPC64_TOC16 47 #define R_PPC64_TOC16_LO 48 #define R_PPC64_TOC16_HI 49 #define R_PPC64_TOC16_HA 50 #define R_PPC64_TOC 51 #define R_PPC64_DTPMOD64 68 #define R_PPC64_TPREL64 73 #define R_PPC64_DTPREL64 78 /* * TLS relocations */ #define R_PPC_TLS 67 #define R_PPC_DTPMOD32 68 #define R_PPC_TPREL16 69 #define R_PPC_TPREL16_LO 70 #define R_PPC_TPREL16_HI 71 #define R_PPC_TPREL16_HA 72 #define R_PPC_TPREL32 73 #define R_PPC_DTPREL16 74 #define R_PPC_DTPREL16_LO 75 #define R_PPC_DTPREL16_HI 76 #define R_PPC_DTPREL16_HA 77 #define R_PPC_DTPREL32 78 #define R_PPC_GOT_TLSGD16 79 #define R_PPC_GOT_TLSGD16_LO 80 #define R_PPC_GOT_TLSGD16_HI 81 #define R_PPC_GOT_TLSGD16_HA 82 #define R_PPC_GOT_TLSLD16 83 #define R_PPC_GOT_TLSLD16_LO 84 #define R_PPC_GOT_TLSLD16_HI 85 #define R_PPC_GOT_TLSLD16_HA 86 #define R_PPC_GOT_TPREL16 87 #define R_PPC_GOT_TPREL16_LO 88 #define R_PPC_GOT_TPREL16_HI 89 #define R_PPC_GOT_TPREL16_HA 90 /* * The remaining relocs are from the Embedded ELF ABI, and are not in the * SVR4 ELF ABI. */ #define R_PPC_EMB_NADDR32 101 #define R_PPC_EMB_NADDR16 102 #define R_PPC_EMB_NADDR16_LO 103 #define R_PPC_EMB_NADDR16_HI 104 #define R_PPC_EMB_NADDR16_HA 105 #define R_PPC_EMB_SDAI16 106 #define R_PPC_EMB_SDA2I16 107 #define R_PPC_EMB_SDA2REL 108 #define R_PPC_EMB_SDA21 109 #define R_PPC_EMB_MRKREF 110 #define R_PPC_EMB_RELSEC16 111 #define R_PPC_EMB_RELST_LO 112 #define R_PPC_EMB_RELST_HI 113 #define R_PPC_EMB_RELST_HA 114 #define R_PPC_EMB_BIT_FLD 115 #define R_PPC_EMB_RELSDA 116 /* * RISC-V relocation types. */ /* Relocation types used by the dynamic linker. */ #define R_RISCV_NONE 0 #define R_RISCV_32 1 #define R_RISCV_64 2 #define R_RISCV_RELATIVE 3 #define R_RISCV_COPY 4 #define R_RISCV_JUMP_SLOT 5 #define R_RISCV_TLS_DTPMOD32 6 #define R_RISCV_TLS_DTPMOD64 7 #define R_RISCV_TLS_DTPREL32 8 #define R_RISCV_TLS_DTPREL64 9 #define R_RISCV_TLS_TPREL32 10 #define R_RISCV_TLS_TPREL64 11 /* Relocation types not used by the dynamic linker. */ #define R_RISCV_BRANCH 16 #define R_RISCV_JAL 17 #define R_RISCV_CALL 18 #define R_RISCV_CALL_PLT 19 #define R_RISCV_GOT_HI20 20 #define R_RISCV_TLS_GOT_HI20 21 #define R_RISCV_TLS_GD_HI20 22 #define R_RISCV_PCREL_HI20 23 #define R_RISCV_PCREL_LO12_I 24 #define R_RISCV_PCREL_LO12_S 25 #define R_RISCV_HI20 26 #define R_RISCV_LO12_I 27 #define R_RISCV_LO12_S 28 #define R_RISCV_TPREL_HI20 29 #define R_RISCV_TPREL_LO12_I 30 #define R_RISCV_TPREL_LO12_S 31 #define R_RISCV_TPREL_ADD 32 #define R_RISCV_ADD8 33 #define R_RISCV_ADD16 34 #define R_RISCV_ADD32 35 #define R_RISCV_ADD64 36 #define R_RISCV_SUB8 37 #define R_RISCV_SUB16 38 #define R_RISCV_SUB32 39 #define R_RISCV_SUB64 40 #define R_RISCV_GNU_VTINHERIT 41 #define R_RISCV_GNU_VTENTRY 42 #define R_RISCV_ALIGN 43 #define R_RISCV_RVC_BRANCH 44 #define R_RISCV_RVC_JUMP 45 #define R_RISCV_RVC_LUI 46 #define R_RISCV_GPREL_I 47 #define R_RISCV_GPREL_S 48 #define R_RISCV_TPREL_I 49 #define R_RISCV_TPREL_S 50 #define R_RISCV_RELAX 51 #define R_RISCV_SUB6 52 #define R_RISCV_SET6 53 #define R_RISCV_SET8 54 #define R_RISCV_SET16 55 #define R_RISCV_SET32 56 #define R_SPARC_NONE 0 #define R_SPARC_8 1 #define R_SPARC_16 2 #define R_SPARC_32 3 #define R_SPARC_DISP8 4 #define R_SPARC_DISP16 5 #define R_SPARC_DISP32 6 #define R_SPARC_WDISP30 7 #define R_SPARC_WDISP22 8 #define R_SPARC_HI22 9 #define R_SPARC_22 10 #define R_SPARC_13 11 #define R_SPARC_LO10 12 #define R_SPARC_GOT10 13 #define R_SPARC_GOT13 14 #define R_SPARC_GOT22 15 #define R_SPARC_PC10 16 #define R_SPARC_PC22 17 #define R_SPARC_WPLT30 18 #define R_SPARC_COPY 19 #define R_SPARC_GLOB_DAT 20 #define R_SPARC_JMP_SLOT 21 #define R_SPARC_RELATIVE 22 #define R_SPARC_UA32 23 #define R_SPARC_PLT32 24 #define R_SPARC_HIPLT22 25 #define R_SPARC_LOPLT10 26 #define R_SPARC_PCPLT32 27 #define R_SPARC_PCPLT22 28 #define R_SPARC_PCPLT10 29 #define R_SPARC_10 30 #define R_SPARC_11 31 #define R_SPARC_64 32 #define R_SPARC_OLO10 33 #define R_SPARC_HH22 34 #define R_SPARC_HM10 35 #define R_SPARC_LM22 36 #define R_SPARC_PC_HH22 37 #define R_SPARC_PC_HM10 38 #define R_SPARC_PC_LM22 39 #define R_SPARC_WDISP16 40 #define R_SPARC_WDISP19 41 #define R_SPARC_GLOB_JMP 42 #define R_SPARC_7 43 #define R_SPARC_5 44 #define R_SPARC_6 45 #define R_SPARC_DISP64 46 #define R_SPARC_PLT64 47 #define R_SPARC_HIX22 48 #define R_SPARC_LOX10 49 #define R_SPARC_H44 50 #define R_SPARC_M44 51 #define R_SPARC_L44 52 #define R_SPARC_REGISTER 53 #define R_SPARC_UA64 54 #define R_SPARC_UA16 55 #define R_SPARC_TLS_GD_HI22 56 #define R_SPARC_TLS_GD_LO10 57 #define R_SPARC_TLS_GD_ADD 58 #define R_SPARC_TLS_GD_CALL 59 #define R_SPARC_TLS_LDM_HI22 60 #define R_SPARC_TLS_LDM_LO10 61 #define R_SPARC_TLS_LDM_ADD 62 #define R_SPARC_TLS_LDM_CALL 63 #define R_SPARC_TLS_LDO_HIX22 64 #define R_SPARC_TLS_LDO_LOX10 65 #define R_SPARC_TLS_LDO_ADD 66 #define R_SPARC_TLS_IE_HI22 67 #define R_SPARC_TLS_IE_LO10 68 #define R_SPARC_TLS_IE_LD 69 #define R_SPARC_TLS_IE_LDX 70 #define R_SPARC_TLS_IE_ADD 71 #define R_SPARC_TLS_LE_HIX22 72 #define R_SPARC_TLS_LE_LOX10 73 #define R_SPARC_TLS_DTPMOD32 74 #define R_SPARC_TLS_DTPMOD64 75 #define R_SPARC_TLS_DTPOFF32 76 #define R_SPARC_TLS_DTPOFF64 77 #define R_SPARC_TLS_TPOFF32 78 #define R_SPARC_TLS_TPOFF64 79 #define R_X86_64_NONE 0 /* No relocation. */ #define R_X86_64_64 1 /* Add 64 bit symbol value. */ #define R_X86_64_PC32 2 /* PC-relative 32 bit signed sym value. */ #define R_X86_64_GOT32 3 /* PC-relative 32 bit GOT offset. */ #define R_X86_64_PLT32 4 /* PC-relative 32 bit PLT offset. */ #define R_X86_64_COPY 5 /* Copy data from shared object. */ #define R_X86_64_GLOB_DAT 6 /* Set GOT entry to data address. */ #define R_X86_64_JMP_SLOT 7 /* Set GOT entry to code address. */ #define R_X86_64_RELATIVE 8 /* Add load address of shared object. */ #define R_X86_64_GOTPCREL 9 /* Add 32 bit signed pcrel offset to GOT. */ #define R_X86_64_32 10 /* Add 32 bit zero extended symbol value */ #define R_X86_64_32S 11 /* Add 32 bit sign extended symbol value */ #define R_X86_64_16 12 /* Add 16 bit zero extended symbol value */ #define R_X86_64_PC16 13 /* Add 16 bit signed extended pc relative symbol value */ #define R_X86_64_8 14 /* Add 8 bit zero extended symbol value */ #define R_X86_64_PC8 15 /* Add 8 bit signed extended pc relative symbol value */ #define R_X86_64_DTPMOD64 16 /* ID of module containing symbol */ #define R_X86_64_DTPOFF64 17 /* Offset in TLS block */ #define R_X86_64_TPOFF64 18 /* Offset in static TLS block */ #define R_X86_64_TLSGD 19 /* PC relative offset to GD GOT entry */ #define R_X86_64_TLSLD 20 /* PC relative offset to LD GOT entry */ #define R_X86_64_DTPOFF32 21 /* Offset in TLS block */ #define R_X86_64_GOTTPOFF 22 /* PC relative offset to IE GOT entry */ #define R_X86_64_TPOFF32 23 /* Offset in static TLS block */ #define R_X86_64_PC64 24 /* PC-relative 64 bit signed sym value. */ #define R_X86_64_GOTOFF64 25 #define R_X86_64_GOTPC32 26 #define R_X86_64_GOT64 27 #define R_X86_64_GOTPCREL64 28 #define R_X86_64_GOTPC64 29 #define R_X86_64_GOTPLT64 30 #define R_X86_64_PLTOFF64 31 #define R_X86_64_SIZE32 32 #define R_X86_64_SIZE64 33 #define R_X86_64_GOTPC32_TLSDESC 34 #define R_X86_64_TLSDESC_CALL 35 #define R_X86_64_TLSDESC 36 #define R_X86_64_IRELATIVE 37 #endif /* !_SYS_ELF_COMMON_H_ */ Index: head/sys/vm/vm_map.c =================================================================== --- head/sys/vm/vm_map.c (revision 354789) +++ head/sys/vm/vm_map.c (revision 354790) @@ -1,4974 +1,4977 @@ /*- * SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU) * * Copyright (c) 1991, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * The Mach Operating System project at Carnegie-Mellon University. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)vm_map.c 8.3 (Berkeley) 1/12/94 * * * Copyright (c) 1987, 1990 Carnegie-Mellon University. * All rights reserved. * * Authors: Avadis Tevanian, Jr., Michael Wayne Young * * Permission to use, copy, modify and distribute this software and * its documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. * * Carnegie Mellon requests users of this software to return to * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 * * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. */ /* * Virtual memory mapping module. */ #include __FBSDID("$FreeBSD$"); #include #include +#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Virtual memory maps provide for the mapping, protection, * and sharing of virtual memory objects. In addition, * this module provides for an efficient virtual copy of * memory from one map to another. * * Synchronization is required prior to most operations. * * Maps consist of an ordered doubly-linked list of simple * entries; a self-adjusting binary search tree of these * entries is used to speed up lookups. * * Since portions of maps are specified by start/end addresses, * which may not align with existing map entries, all * routines merely "clip" entries to these start/end values. * [That is, an entry is split into two, bordering at a * start or end value.] Note that these clippings may not * always be necessary (as the two resulting entries are then * not changed); however, the clipping is done for convenience. * * As mentioned above, virtual copy operations are performed * by copying VM object references from one map to * another, and then marking both regions as copy-on-write. */ static struct mtx map_sleep_mtx; static uma_zone_t mapentzone; static uma_zone_t kmapentzone; static uma_zone_t mapzone; static uma_zone_t vmspace_zone; static int vmspace_zinit(void *mem, int size, int flags); static int vm_map_zinit(void *mem, int ize, int flags); static void _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max); static void vm_map_entry_deallocate(vm_map_entry_t entry, boolean_t system_map); static void vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry); static void vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry); static int vm_map_growstack(vm_map_t map, vm_offset_t addr, vm_map_entry_t gap_entry); static void vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot, vm_object_t object, vm_pindex_t pindex, vm_size_t size, int flags); #ifdef INVARIANTS static void vm_map_zdtor(void *mem, int size, void *arg); static void vmspace_zdtor(void *mem, int size, void *arg); #endif static int vm_map_stack_locked(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize, vm_size_t growsize, vm_prot_t prot, vm_prot_t max, int cow); static void vm_map_wire_entry_failure(vm_map_t map, vm_map_entry_t entry, vm_offset_t failed_addr); #define ENTRY_CHARGED(e) ((e)->cred != NULL || \ ((e)->object.vm_object != NULL && (e)->object.vm_object->cred != NULL && \ !((e)->eflags & MAP_ENTRY_NEEDS_COPY))) /* * PROC_VMSPACE_{UN,}LOCK() can be a noop as long as vmspaces are type * stable. */ #define PROC_VMSPACE_LOCK(p) do { } while (0) #define PROC_VMSPACE_UNLOCK(p) do { } while (0) /* * VM_MAP_RANGE_CHECK: [ internal use only ] * * Asserts that the starting and ending region * addresses fall within the valid range of the map. */ #define VM_MAP_RANGE_CHECK(map, start, end) \ { \ if (start < vm_map_min(map)) \ start = vm_map_min(map); \ if (end > vm_map_max(map)) \ end = vm_map_max(map); \ if (start > end) \ start = end; \ } /* * vm_map_startup: * * Initialize the vm_map module. Must be called before * any other vm_map routines. * * Map and entry structures are allocated from the general * purpose memory pool with some exceptions: * * - The kernel map and kmem submap are allocated statically. * - Kernel map entries are allocated out of a static pool. * * These restrictions are necessary since malloc() uses the * maps and requires map entries. */ void vm_map_startup(void) { mtx_init(&map_sleep_mtx, "vm map sleep mutex", NULL, MTX_DEF); mapzone = uma_zcreate("MAP", sizeof(struct vm_map), NULL, #ifdef INVARIANTS vm_map_zdtor, #else NULL, #endif vm_map_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); uma_prealloc(mapzone, MAX_KMAP); kmapentzone = uma_zcreate("KMAP ENTRY", sizeof(struct vm_map_entry), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_MTXCLASS | UMA_ZONE_VM); mapentzone = uma_zcreate("MAP ENTRY", sizeof(struct vm_map_entry), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); vmspace_zone = uma_zcreate("VMSPACE", sizeof(struct vmspace), NULL, #ifdef INVARIANTS vmspace_zdtor, #else NULL, #endif vmspace_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); } static int vmspace_zinit(void *mem, int size, int flags) { struct vmspace *vm; vm = (struct vmspace *)mem; vm->vm_map.pmap = NULL; (void)vm_map_zinit(&vm->vm_map, sizeof(vm->vm_map), flags); PMAP_LOCK_INIT(vmspace_pmap(vm)); return (0); } static int vm_map_zinit(void *mem, int size, int flags) { vm_map_t map; map = (vm_map_t)mem; memset(map, 0, sizeof(*map)); mtx_init(&map->system_mtx, "vm map (system)", NULL, MTX_DEF | MTX_DUPOK); sx_init(&map->lock, "vm map (user)"); return (0); } #ifdef INVARIANTS static void vmspace_zdtor(void *mem, int size, void *arg) { struct vmspace *vm; vm = (struct vmspace *)mem; vm_map_zdtor(&vm->vm_map, sizeof(vm->vm_map), arg); } static void vm_map_zdtor(void *mem, int size, void *arg) { vm_map_t map; map = (vm_map_t)mem; KASSERT(map->nentries == 0, ("map %p nentries == %d on free.", map, map->nentries)); KASSERT(map->size == 0, ("map %p size == %lu on free.", map, (unsigned long)map->size)); } #endif /* INVARIANTS */ /* * Allocate a vmspace structure, including a vm_map and pmap, * and initialize those structures. The refcnt is set to 1. * * If 'pinit' is NULL then the embedded pmap is initialized via pmap_pinit(). */ struct vmspace * vmspace_alloc(vm_offset_t min, vm_offset_t max, pmap_pinit_t pinit) { struct vmspace *vm; vm = uma_zalloc(vmspace_zone, M_WAITOK); KASSERT(vm->vm_map.pmap == NULL, ("vm_map.pmap must be NULL")); if (!pinit(vmspace_pmap(vm))) { uma_zfree(vmspace_zone, vm); return (NULL); } CTR1(KTR_VM, "vmspace_alloc: %p", vm); _vm_map_init(&vm->vm_map, vmspace_pmap(vm), min, max); vm->vm_refcnt = 1; vm->vm_shm = NULL; vm->vm_swrss = 0; vm->vm_tsize = 0; vm->vm_dsize = 0; vm->vm_ssize = 0; vm->vm_taddr = 0; vm->vm_daddr = 0; vm->vm_maxsaddr = 0; return (vm); } #ifdef RACCT static void vmspace_container_reset(struct proc *p) { PROC_LOCK(p); racct_set(p, RACCT_DATA, 0); racct_set(p, RACCT_STACK, 0); racct_set(p, RACCT_RSS, 0); racct_set(p, RACCT_MEMLOCK, 0); racct_set(p, RACCT_VMEM, 0); PROC_UNLOCK(p); } #endif static inline void vmspace_dofree(struct vmspace *vm) { CTR1(KTR_VM, "vmspace_free: %p", vm); /* * Make sure any SysV shm is freed, it might not have been in * exit1(). */ shmexit(vm); /* * Lock the map, to wait out all other references to it. * Delete all of the mappings and pages they hold, then call * the pmap module to reclaim anything left. */ (void)vm_map_remove(&vm->vm_map, vm_map_min(&vm->vm_map), vm_map_max(&vm->vm_map)); pmap_release(vmspace_pmap(vm)); vm->vm_map.pmap = NULL; uma_zfree(vmspace_zone, vm); } void vmspace_free(struct vmspace *vm) { WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "vmspace_free() called"); if (vm->vm_refcnt == 0) panic("vmspace_free: attempt to free already freed vmspace"); if (atomic_fetchadd_int(&vm->vm_refcnt, -1) == 1) vmspace_dofree(vm); } void vmspace_exitfree(struct proc *p) { struct vmspace *vm; PROC_VMSPACE_LOCK(p); vm = p->p_vmspace; p->p_vmspace = NULL; PROC_VMSPACE_UNLOCK(p); KASSERT(vm == &vmspace0, ("vmspace_exitfree: wrong vmspace")); vmspace_free(vm); } void vmspace_exit(struct thread *td) { int refcnt; struct vmspace *vm; struct proc *p; /* * Release user portion of address space. * This releases references to vnodes, * which could cause I/O if the file has been unlinked. * Need to do this early enough that we can still sleep. * * The last exiting process to reach this point releases as * much of the environment as it can. vmspace_dofree() is the * slower fallback in case another process had a temporary * reference to the vmspace. */ p = td->td_proc; vm = p->p_vmspace; atomic_add_int(&vmspace0.vm_refcnt, 1); refcnt = vm->vm_refcnt; do { if (refcnt > 1 && p->p_vmspace != &vmspace0) { /* Switch now since other proc might free vmspace */ PROC_VMSPACE_LOCK(p); p->p_vmspace = &vmspace0; PROC_VMSPACE_UNLOCK(p); pmap_activate(td); } } while (!atomic_fcmpset_int(&vm->vm_refcnt, &refcnt, refcnt - 1)); if (refcnt == 1) { if (p->p_vmspace != vm) { /* vmspace not yet freed, switch back */ PROC_VMSPACE_LOCK(p); p->p_vmspace = vm; PROC_VMSPACE_UNLOCK(p); pmap_activate(td); } pmap_remove_pages(vmspace_pmap(vm)); /* Switch now since this proc will free vmspace */ PROC_VMSPACE_LOCK(p); p->p_vmspace = &vmspace0; PROC_VMSPACE_UNLOCK(p); pmap_activate(td); vmspace_dofree(vm); } #ifdef RACCT if (racct_enable) vmspace_container_reset(p); #endif } /* Acquire reference to vmspace owned by another process. */ struct vmspace * vmspace_acquire_ref(struct proc *p) { struct vmspace *vm; int refcnt; PROC_VMSPACE_LOCK(p); vm = p->p_vmspace; if (vm == NULL) { PROC_VMSPACE_UNLOCK(p); return (NULL); } refcnt = vm->vm_refcnt; do { if (refcnt <= 0) { /* Avoid 0->1 transition */ PROC_VMSPACE_UNLOCK(p); return (NULL); } } while (!atomic_fcmpset_int(&vm->vm_refcnt, &refcnt, refcnt + 1)); if (vm != p->p_vmspace) { PROC_VMSPACE_UNLOCK(p); vmspace_free(vm); return (NULL); } PROC_VMSPACE_UNLOCK(p); return (vm); } /* * Switch between vmspaces in an AIO kernel process. * * The new vmspace is either the vmspace of a user process obtained * from an active AIO request or the initial vmspace of the AIO kernel * process (when it is idling). Because user processes will block to * drain any active AIO requests before proceeding in exit() or * execve(), the reference count for vmspaces from AIO requests can * never be 0. Similarly, AIO kernel processes hold an extra * reference on their initial vmspace for the life of the process. As * a result, the 'newvm' vmspace always has a non-zero reference * count. This permits an additional reference on 'newvm' to be * acquired via a simple atomic increment rather than the loop in * vmspace_acquire_ref() above. */ void vmspace_switch_aio(struct vmspace *newvm) { struct vmspace *oldvm; /* XXX: Need some way to assert that this is an aio daemon. */ KASSERT(newvm->vm_refcnt > 0, ("vmspace_switch_aio: newvm unreferenced")); oldvm = curproc->p_vmspace; if (oldvm == newvm) return; /* * Point to the new address space and refer to it. */ curproc->p_vmspace = newvm; atomic_add_int(&newvm->vm_refcnt, 1); /* Activate the new mapping. */ pmap_activate(curthread); vmspace_free(oldvm); } void _vm_map_lock(vm_map_t map, const char *file, int line) { if (map->system_map) mtx_lock_flags_(&map->system_mtx, 0, file, line); else sx_xlock_(&map->lock, file, line); map->timestamp++; } void vm_map_entry_set_vnode_text(vm_map_entry_t entry, bool add) { vm_object_t object, object1; struct vnode *vp; if ((entry->eflags & MAP_ENTRY_VN_EXEC) == 0) return; KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0, ("Submap with execs")); object = entry->object.vm_object; KASSERT(object != NULL, ("No object for text, entry %p", entry)); VM_OBJECT_RLOCK(object); while ((object1 = object->backing_object) != NULL) { VM_OBJECT_RLOCK(object1); VM_OBJECT_RUNLOCK(object); object = object1; } vp = NULL; if (object->type == OBJT_DEAD) { /* * For OBJT_DEAD objects, v_writecount was handled in * vnode_pager_dealloc(). */ } else if (object->type == OBJT_VNODE) { vp = object->handle; } else if (object->type == OBJT_SWAP) { KASSERT((object->flags & OBJ_TMPFS_NODE) != 0, ("vm_map_entry_set_vnode_text: swap and !TMPFS " "entry %p, object %p, add %d", entry, object, add)); /* * Tmpfs VREG node, which was reclaimed, has * OBJ_TMPFS_NODE flag set, but not OBJ_TMPFS. In * this case there is no v_writecount to adjust. */ if ((object->flags & OBJ_TMPFS) != 0) vp = object->un_pager.swp.swp_tmpfs; } else { KASSERT(0, ("vm_map_entry_set_vnode_text: wrong object type, " "entry %p, object %p, add %d", entry, object, add)); } if (vp != NULL) { if (add) { VOP_SET_TEXT_CHECKED(vp); VM_OBJECT_RUNLOCK(object); } else { vhold(vp); VM_OBJECT_RUNLOCK(object); vn_lock(vp, LK_SHARED | LK_RETRY); VOP_UNSET_TEXT_CHECKED(vp); VOP_UNLOCK(vp, 0); vdrop(vp); } } else { VM_OBJECT_RUNLOCK(object); } } /* * Use a different name for this vm_map_entry field when it's use * is not consistent with its use as part of an ordered search tree. */ #define defer_next right static void vm_map_process_deferred(void) { struct thread *td; vm_map_entry_t entry, next; vm_object_t object; td = curthread; entry = td->td_map_def_user; td->td_map_def_user = NULL; while (entry != NULL) { next = entry->defer_next; MPASS((entry->eflags & (MAP_ENTRY_WRITECNT | MAP_ENTRY_VN_EXEC)) != (MAP_ENTRY_WRITECNT | MAP_ENTRY_VN_EXEC)); if ((entry->eflags & MAP_ENTRY_WRITECNT) != 0) { /* * Decrement the object's writemappings and * possibly the vnode's v_writecount. */ KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0, ("Submap with writecount")); object = entry->object.vm_object; KASSERT(object != NULL, ("No object for writecount")); vm_pager_release_writecount(object, entry->start, entry->end); } vm_map_entry_set_vnode_text(entry, false); vm_map_entry_deallocate(entry, FALSE); entry = next; } } #ifdef INVARIANTS static void _vm_map_assert_locked(vm_map_t map, const char *file, int line) { if (map->system_map) mtx_assert_(&map->system_mtx, MA_OWNED, file, line); else sx_assert_(&map->lock, SA_XLOCKED, file, line); } #define VM_MAP_ASSERT_LOCKED(map) \ _vm_map_assert_locked(map, LOCK_FILE, LOCK_LINE) enum { VMMAP_CHECK_NONE, VMMAP_CHECK_UNLOCK, VMMAP_CHECK_ALL }; #ifdef DIAGNOSTIC static int enable_vmmap_check = VMMAP_CHECK_UNLOCK; #else static int enable_vmmap_check = VMMAP_CHECK_NONE; #endif SYSCTL_INT(_debug, OID_AUTO, vmmap_check, CTLFLAG_RWTUN, &enable_vmmap_check, 0, "Enable vm map consistency checking"); static void _vm_map_assert_consistent(vm_map_t map, int check); #define VM_MAP_ASSERT_CONSISTENT(map) \ _vm_map_assert_consistent(map, VMMAP_CHECK_ALL) #ifdef DIAGNOSTIC #define VM_MAP_UNLOCK_CONSISTENT(map) do { \ if (map->nupdates > map->nentries) { \ _vm_map_assert_consistent(map, VMMAP_CHECK_UNLOCK); \ map->nupdates = 0; \ } \ } while (0) #else #define VM_MAP_UNLOCK_CONSISTENT(map) #endif #else #define VM_MAP_ASSERT_LOCKED(map) #define VM_MAP_ASSERT_CONSISTENT(map) #define VM_MAP_UNLOCK_CONSISTENT(map) #endif /* INVARIANTS */ void _vm_map_unlock(vm_map_t map, const char *file, int line) { VM_MAP_UNLOCK_CONSISTENT(map); if (map->system_map) mtx_unlock_flags_(&map->system_mtx, 0, file, line); else { sx_xunlock_(&map->lock, file, line); vm_map_process_deferred(); } } void _vm_map_lock_read(vm_map_t map, const char *file, int line) { if (map->system_map) mtx_lock_flags_(&map->system_mtx, 0, file, line); else sx_slock_(&map->lock, file, line); } void _vm_map_unlock_read(vm_map_t map, const char *file, int line) { if (map->system_map) mtx_unlock_flags_(&map->system_mtx, 0, file, line); else { sx_sunlock_(&map->lock, file, line); vm_map_process_deferred(); } } int _vm_map_trylock(vm_map_t map, const char *file, int line) { int error; error = map->system_map ? !mtx_trylock_flags_(&map->system_mtx, 0, file, line) : !sx_try_xlock_(&map->lock, file, line); if (error == 0) map->timestamp++; return (error == 0); } int _vm_map_trylock_read(vm_map_t map, const char *file, int line) { int error; error = map->system_map ? !mtx_trylock_flags_(&map->system_mtx, 0, file, line) : !sx_try_slock_(&map->lock, file, line); return (error == 0); } /* * _vm_map_lock_upgrade: [ internal use only ] * * Tries to upgrade a read (shared) lock on the specified map to a write * (exclusive) lock. Returns the value "0" if the upgrade succeeds and a * non-zero value if the upgrade fails. If the upgrade fails, the map is * returned without a read or write lock held. * * Requires that the map be read locked. */ int _vm_map_lock_upgrade(vm_map_t map, const char *file, int line) { unsigned int last_timestamp; if (map->system_map) { mtx_assert_(&map->system_mtx, MA_OWNED, file, line); } else { if (!sx_try_upgrade_(&map->lock, file, line)) { last_timestamp = map->timestamp; sx_sunlock_(&map->lock, file, line); vm_map_process_deferred(); /* * If the map's timestamp does not change while the * map is unlocked, then the upgrade succeeds. */ sx_xlock_(&map->lock, file, line); if (last_timestamp != map->timestamp) { sx_xunlock_(&map->lock, file, line); return (1); } } } map->timestamp++; return (0); } void _vm_map_lock_downgrade(vm_map_t map, const char *file, int line) { if (map->system_map) { mtx_assert_(&map->system_mtx, MA_OWNED, file, line); } else { VM_MAP_UNLOCK_CONSISTENT(map); sx_downgrade_(&map->lock, file, line); } } /* * vm_map_locked: * * Returns a non-zero value if the caller holds a write (exclusive) lock * on the specified map and the value "0" otherwise. */ int vm_map_locked(vm_map_t map) { if (map->system_map) return (mtx_owned(&map->system_mtx)); else return (sx_xlocked(&map->lock)); } /* * _vm_map_unlock_and_wait: * * Atomically releases the lock on the specified map and puts the calling * thread to sleep. The calling thread will remain asleep until either * vm_map_wakeup() is performed on the map or the specified timeout is * exceeded. * * WARNING! This function does not perform deferred deallocations of * objects and map entries. Therefore, the calling thread is expected to * reacquire the map lock after reawakening and later perform an ordinary * unlock operation, such as vm_map_unlock(), before completing its * operation on the map. */ int _vm_map_unlock_and_wait(vm_map_t map, int timo, const char *file, int line) { VM_MAP_UNLOCK_CONSISTENT(map); mtx_lock(&map_sleep_mtx); if (map->system_map) mtx_unlock_flags_(&map->system_mtx, 0, file, line); else sx_xunlock_(&map->lock, file, line); return (msleep(&map->root, &map_sleep_mtx, PDROP | PVM, "vmmaps", timo)); } /* * vm_map_wakeup: * * Awaken any threads that have slept on the map using * vm_map_unlock_and_wait(). */ void vm_map_wakeup(vm_map_t map) { /* * Acquire and release map_sleep_mtx to prevent a wakeup() * from being performed (and lost) between the map unlock * and the msleep() in _vm_map_unlock_and_wait(). */ mtx_lock(&map_sleep_mtx); mtx_unlock(&map_sleep_mtx); wakeup(&map->root); } void vm_map_busy(vm_map_t map) { VM_MAP_ASSERT_LOCKED(map); map->busy++; } void vm_map_unbusy(vm_map_t map) { VM_MAP_ASSERT_LOCKED(map); KASSERT(map->busy, ("vm_map_unbusy: not busy")); if (--map->busy == 0 && (map->flags & MAP_BUSY_WAKEUP)) { vm_map_modflags(map, 0, MAP_BUSY_WAKEUP); wakeup(&map->busy); } } void vm_map_wait_busy(vm_map_t map) { VM_MAP_ASSERT_LOCKED(map); while (map->busy) { vm_map_modflags(map, MAP_BUSY_WAKEUP, 0); if (map->system_map) msleep(&map->busy, &map->system_mtx, 0, "mbusy", 0); else sx_sleep(&map->busy, &map->lock, 0, "mbusy", 0); } map->timestamp++; } long vmspace_resident_count(struct vmspace *vmspace) { return pmap_resident_count(vmspace_pmap(vmspace)); } /* * vm_map_create: * * Creates and returns a new empty VM map with * the given physical map structure, and having * the given lower and upper address bounds. */ vm_map_t vm_map_create(pmap_t pmap, vm_offset_t min, vm_offset_t max) { vm_map_t result; result = uma_zalloc(mapzone, M_WAITOK); CTR1(KTR_VM, "vm_map_create: %p", result); _vm_map_init(result, pmap, min, max); return (result); } /* * Initialize an existing vm_map structure * such as that in the vmspace structure. */ static void _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max) { map->header.next = map->header.prev = &map->header; map->header.eflags = MAP_ENTRY_HEADER; map->needs_wakeup = FALSE; map->system_map = 0; map->pmap = pmap; map->header.end = min; map->header.start = max; map->flags = 0; map->root = NULL; map->timestamp = 0; map->busy = 0; map->anon_loc = 0; #ifdef DIAGNOSTIC map->nupdates = 0; #endif } void vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max) { _vm_map_init(map, pmap, min, max); mtx_init(&map->system_mtx, "system map", NULL, MTX_DEF | MTX_DUPOK); sx_init(&map->lock, "user map"); } /* * vm_map_entry_dispose: [ internal use only ] * * Inverse of vm_map_entry_create. */ static void vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry) { uma_zfree(map->system_map ? kmapentzone : mapentzone, entry); } /* * vm_map_entry_create: [ internal use only ] * * Allocates a VM map entry for insertion. * No entry fields are filled in. */ static vm_map_entry_t vm_map_entry_create(vm_map_t map) { vm_map_entry_t new_entry; if (map->system_map) new_entry = uma_zalloc(kmapentzone, M_NOWAIT); else new_entry = uma_zalloc(mapentzone, M_WAITOK); if (new_entry == NULL) panic("vm_map_entry_create: kernel resources exhausted"); return (new_entry); } /* * vm_map_entry_set_behavior: * * Set the expected access behavior, either normal, random, or * sequential. */ static inline void vm_map_entry_set_behavior(vm_map_entry_t entry, u_char behavior) { entry->eflags = (entry->eflags & ~MAP_ENTRY_BEHAV_MASK) | (behavior & MAP_ENTRY_BEHAV_MASK); } /* * vm_map_entry_max_free_{left,right}: * * Compute the size of the largest free gap between two entries, * one the root of a tree and the other the ancestor of that root * that is the least or greatest ancestor found on the search path. */ static inline vm_size_t vm_map_entry_max_free_left(vm_map_entry_t root, vm_map_entry_t left_ancestor) { return (root->left != NULL ? root->left->max_free : root->start - left_ancestor->end); } static inline vm_size_t vm_map_entry_max_free_right(vm_map_entry_t root, vm_map_entry_t right_ancestor) { return (root->right != NULL ? root->right->max_free : right_ancestor->start - root->end); } #define SPLAY_LEFT_STEP(root, y, rlist, test) do { \ vm_size_t max_free; \ \ /* \ * Infer root->right->max_free == root->max_free when \ * y->max_free < root->max_free || root->max_free == 0. \ * Otherwise, look right to find it. \ */ \ y = root->left; \ max_free = root->max_free; \ KASSERT(max_free >= vm_map_entry_max_free_right(root, rlist), \ ("%s: max_free invariant fails", __func__)); \ if (y == NULL ? max_free > 0 : max_free - 1 < y->max_free) \ max_free = vm_map_entry_max_free_right(root, rlist); \ if (y != NULL && (test)) { \ /* Rotate right and make y root. */ \ root->left = y->right; \ y->right = root; \ if (max_free < y->max_free) \ root->max_free = max_free = MAX(max_free, \ vm_map_entry_max_free_left(root, y)); \ root = y; \ y = root->left; \ } \ /* Copy right->max_free. Put root on rlist. */ \ root->max_free = max_free; \ KASSERT(max_free == vm_map_entry_max_free_right(root, rlist), \ ("%s: max_free not copied from right", __func__)); \ root->left = rlist; \ rlist = root; \ root = y; \ } while (0) #define SPLAY_RIGHT_STEP(root, y, llist, test) do { \ vm_size_t max_free; \ \ /* \ * Infer root->left->max_free == root->max_free when \ * y->max_free < root->max_free || root->max_free == 0. \ * Otherwise, look left to find it. \ */ \ y = root->right; \ max_free = root->max_free; \ KASSERT(max_free >= vm_map_entry_max_free_left(root, llist), \ ("%s: max_free invariant fails", __func__)); \ if (y == NULL ? max_free > 0 : max_free - 1 < y->max_free) \ max_free = vm_map_entry_max_free_left(root, llist); \ if (y != NULL && (test)) { \ /* Rotate left and make y root. */ \ root->right = y->left; \ y->left = root; \ if (max_free < y->max_free) \ root->max_free = max_free = MAX(max_free, \ vm_map_entry_max_free_right(root, y)); \ root = y; \ y = root->right; \ } \ /* Copy left->max_free. Put root on llist. */ \ root->max_free = max_free; \ KASSERT(max_free == vm_map_entry_max_free_left(root, llist), \ ("%s: max_free not copied from left", __func__)); \ root->right = llist; \ llist = root; \ root = y; \ } while (0) /* * Walk down the tree until we find addr or a NULL pointer where addr would go, * breaking off left and right subtrees of nodes less than, or greater than * addr. Treat pointers to nodes with max_free < length as NULL pointers. * llist and rlist are the two sides in reverse order (bottom-up), with llist * linked by the right pointer and rlist linked by the left pointer in the * vm_map_entry, and both lists terminated by &map->header. This function, and * the subsequent call to vm_map_splay_merge, rely on the start and end address * values in &map->header. */ static vm_map_entry_t vm_map_splay_split(vm_map_t map, vm_offset_t addr, vm_size_t length, vm_map_entry_t *out_llist, vm_map_entry_t *out_rlist) { vm_map_entry_t llist, rlist, root, y; llist = rlist = &map->header; root = map->root; while (root != NULL && root->max_free >= length) { KASSERT(llist->end <= root->start && root->end <= rlist->start, ("%s: root not within tree bounds", __func__)); if (addr < root->start) { SPLAY_LEFT_STEP(root, y, rlist, y->max_free >= length && addr < y->start); } else if (addr >= root->end) { SPLAY_RIGHT_STEP(root, y, llist, y->max_free >= length && addr >= y->end); } else break; } *out_llist = llist; *out_rlist = rlist; return (root); } static void vm_map_splay_findnext(vm_map_entry_t root, vm_map_entry_t *iolist) { vm_map_entry_t rlist, y; root = root->right; rlist = *iolist; while (root != NULL) SPLAY_LEFT_STEP(root, y, rlist, true); *iolist = rlist; } static void vm_map_splay_findprev(vm_map_entry_t root, vm_map_entry_t *iolist) { vm_map_entry_t llist, y; root = root->left; llist = *iolist; while (root != NULL) SPLAY_RIGHT_STEP(root, y, llist, true); *iolist = llist; } static inline void vm_map_entry_swap(vm_map_entry_t *a, vm_map_entry_t *b) { vm_map_entry_t tmp; tmp = *b; *b = *a; *a = tmp; } /* * Walk back up the two spines, flip the pointers and set max_free. The * subtrees of the root go at the bottom of llist and rlist. */ static void vm_map_splay_merge(vm_map_t map, vm_map_entry_t root, vm_map_entry_t llist, vm_map_entry_t rlist) { vm_map_entry_t prev; vm_size_t max_free_left, max_free_right; max_free_left = vm_map_entry_max_free_left(root, llist); if (llist != &map->header) { prev = root->left; do { /* * The max_free values of the children of llist are in * llist->max_free and max_free_left. Update with the * max value. */ llist->max_free = max_free_left = MAX(llist->max_free, max_free_left); vm_map_entry_swap(&llist->right, &prev); vm_map_entry_swap(&prev, &llist); } while (llist != &map->header); root->left = prev; } max_free_right = vm_map_entry_max_free_right(root, rlist); if (rlist != &map->header) { prev = root->right; do { /* * The max_free values of the children of rlist are in * rlist->max_free and max_free_right. Update with the * max value. */ rlist->max_free = max_free_right = MAX(rlist->max_free, max_free_right); vm_map_entry_swap(&rlist->left, &prev); vm_map_entry_swap(&prev, &rlist); } while (rlist != &map->header); root->right = prev; } root->max_free = MAX(max_free_left, max_free_right); map->root = root; #ifdef DIAGNOSTIC ++map->nupdates; #endif } /* * vm_map_splay: * * The Sleator and Tarjan top-down splay algorithm with the * following variation. Max_free must be computed bottom-up, so * on the downward pass, maintain the left and right spines in * reverse order. Then, make a second pass up each side to fix * the pointers and compute max_free. The time bound is O(log n) * amortized. * * The new root is the vm_map_entry containing "addr", or else an * adjacent entry (lower if possible) if addr is not in the tree. * * The map must be locked, and leaves it so. * * Returns: the new root. */ static vm_map_entry_t vm_map_splay(vm_map_t map, vm_offset_t addr) { vm_map_entry_t llist, rlist, root; root = vm_map_splay_split(map, addr, 0, &llist, &rlist); if (root != NULL) { /* do nothing */ } else if (llist != &map->header) { /* * Recover the greatest node in the left * subtree and make it the root. */ root = llist; llist = root->right; root->right = NULL; } else if (rlist != &map->header) { /* * Recover the least node in the right * subtree and make it the root. */ root = rlist; rlist = root->left; root->left = NULL; } else { /* There is no root. */ return (NULL); } vm_map_splay_merge(map, root, llist, rlist); VM_MAP_ASSERT_CONSISTENT(map); return (root); } /* * vm_map_entry_{un,}link: * * Insert/remove entries from maps. */ static void vm_map_entry_link(vm_map_t map, vm_map_entry_t entry) { vm_map_entry_t llist, rlist, root; CTR3(KTR_VM, "vm_map_entry_link: map %p, nentries %d, entry %p", map, map->nentries, entry); VM_MAP_ASSERT_LOCKED(map); map->nentries++; root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); KASSERT(root == NULL, ("vm_map_entry_link: link object already mapped")); entry->prev = llist; entry->next = rlist; llist->next = rlist->prev = entry; entry->left = entry->right = NULL; vm_map_splay_merge(map, entry, llist, rlist); VM_MAP_ASSERT_CONSISTENT(map); } enum unlink_merge_type { UNLINK_MERGE_NONE, UNLINK_MERGE_NEXT }; static void vm_map_entry_unlink(vm_map_t map, vm_map_entry_t entry, enum unlink_merge_type op) { vm_map_entry_t llist, rlist, root, y; VM_MAP_ASSERT_LOCKED(map); root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); KASSERT(root != NULL, ("vm_map_entry_unlink: unlink object not mapped")); vm_map_splay_findnext(root, &rlist); switch (op) { case UNLINK_MERGE_NEXT: rlist->start = root->start; rlist->offset = root->offset; y = root->left; root = rlist; rlist = root->left; root->left = y; break; case UNLINK_MERGE_NONE: vm_map_splay_findprev(root, &llist); if (llist != &map->header) { root = llist; llist = root->right; root->right = NULL; } else if (rlist != &map->header) { root = rlist; rlist = root->left; root->left = NULL; } else root = NULL; break; } y = entry->next; y->prev = entry->prev; y->prev->next = y; if (root != NULL) vm_map_splay_merge(map, root, llist, rlist); else map->root = NULL; VM_MAP_ASSERT_CONSISTENT(map); map->nentries--; CTR3(KTR_VM, "vm_map_entry_unlink: map %p, nentries %d, entry %p", map, map->nentries, entry); } /* * vm_map_entry_resize: * * Resize a vm_map_entry, recompute the amount of free space that * follows it and propagate that value up the tree. * * The map must be locked, and leaves it so. */ static void vm_map_entry_resize(vm_map_t map, vm_map_entry_t entry, vm_size_t grow_amount) { vm_map_entry_t llist, rlist, root; VM_MAP_ASSERT_LOCKED(map); root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); KASSERT(root != NULL, ("%s: resize object not mapped", __func__)); vm_map_splay_findnext(root, &rlist); root->right = NULL; entry->end += grow_amount; vm_map_splay_merge(map, root, llist, rlist); VM_MAP_ASSERT_CONSISTENT(map); CTR4(KTR_VM, "%s: map %p, nentries %d, entry %p", __func__, map, map->nentries, entry); } /* * vm_map_lookup_entry: [ internal use only ] * * Finds the map entry containing (or * immediately preceding) the specified address * in the given map; the entry is returned * in the "entry" parameter. The boolean * result indicates whether the address is * actually contained in the map. */ boolean_t vm_map_lookup_entry( vm_map_t map, vm_offset_t address, vm_map_entry_t *entry) /* OUT */ { vm_map_entry_t cur, lbound; boolean_t locked; /* * If the map is empty, then the map entry immediately preceding * "address" is the map's header. */ cur = map->root; if (cur == NULL) { *entry = &map->header; return (FALSE); } if (address >= cur->start && cur->end > address) { *entry = cur; return (TRUE); } if ((locked = vm_map_locked(map)) || sx_try_upgrade(&map->lock)) { /* * Splay requires a write lock on the map. However, it only * restructures the binary search tree; it does not otherwise * change the map. Thus, the map's timestamp need not change * on a temporary upgrade. */ cur = vm_map_splay(map, address); if (!locked) { VM_MAP_UNLOCK_CONSISTENT(map); sx_downgrade(&map->lock); } /* * If "address" is contained within a map entry, the new root * is that map entry. Otherwise, the new root is a map entry * immediately before or after "address". */ if (address < cur->start) { *entry = &map->header; return (FALSE); } *entry = cur; return (address < cur->end); } /* * Since the map is only locked for read access, perform a * standard binary search tree lookup for "address". */ lbound = &map->header; do { if (address < cur->start) { cur = cur->left; } else if (cur->end <= address) { lbound = cur; cur = cur->right; } else { *entry = cur; return (TRUE); } } while (cur != NULL); *entry = lbound; return (FALSE); } /* * vm_map_insert: * * Inserts the given whole VM object into the target * map at the specified address range. The object's * size should match that of the address range. * * Requires that the map be locked, and leaves it so. * * If object is non-NULL, ref count must be bumped by caller * prior to making call to account for the new entry. */ int vm_map_insert(vm_map_t map, vm_object_t object, vm_ooffset_t offset, vm_offset_t start, vm_offset_t end, vm_prot_t prot, vm_prot_t max, int cow) { vm_map_entry_t new_entry, prev_entry; struct ucred *cred; vm_eflags_t protoeflags; vm_inherit_t inheritance; VM_MAP_ASSERT_LOCKED(map); KASSERT(object != kernel_object || (cow & MAP_COPY_ON_WRITE) == 0, ("vm_map_insert: kernel object and COW")); KASSERT(object == NULL || (cow & MAP_NOFAULT) == 0, ("vm_map_insert: paradoxical MAP_NOFAULT request")); KASSERT((prot & ~max) == 0, ("prot %#x is not subset of max_prot %#x", prot, max)); /* * Check that the start and end points are not bogus. */ if (start < vm_map_min(map) || end > vm_map_max(map) || start >= end) return (KERN_INVALID_ADDRESS); /* * Find the entry prior to the proposed starting address; if it's part * of an existing entry, this range is bogus. */ if (vm_map_lookup_entry(map, start, &prev_entry)) return (KERN_NO_SPACE); /* * Assert that the next entry doesn't overlap the end point. */ if (vm_map_entry_succ(prev_entry)->start < end) return (KERN_NO_SPACE); if ((cow & MAP_CREATE_GUARD) != 0 && (object != NULL || max != VM_PROT_NONE)) return (KERN_INVALID_ARGUMENT); protoeflags = 0; if (cow & MAP_COPY_ON_WRITE) protoeflags |= MAP_ENTRY_COW | MAP_ENTRY_NEEDS_COPY; if (cow & MAP_NOFAULT) protoeflags |= MAP_ENTRY_NOFAULT; if (cow & MAP_DISABLE_SYNCER) protoeflags |= MAP_ENTRY_NOSYNC; if (cow & MAP_DISABLE_COREDUMP) protoeflags |= MAP_ENTRY_NOCOREDUMP; if (cow & MAP_STACK_GROWS_DOWN) protoeflags |= MAP_ENTRY_GROWS_DOWN; if (cow & MAP_STACK_GROWS_UP) protoeflags |= MAP_ENTRY_GROWS_UP; if (cow & MAP_WRITECOUNT) protoeflags |= MAP_ENTRY_WRITECNT; if (cow & MAP_VN_EXEC) protoeflags |= MAP_ENTRY_VN_EXEC; if ((cow & MAP_CREATE_GUARD) != 0) protoeflags |= MAP_ENTRY_GUARD; if ((cow & MAP_CREATE_STACK_GAP_DN) != 0) protoeflags |= MAP_ENTRY_STACK_GAP_DN; if ((cow & MAP_CREATE_STACK_GAP_UP) != 0) protoeflags |= MAP_ENTRY_STACK_GAP_UP; if (cow & MAP_INHERIT_SHARE) inheritance = VM_INHERIT_SHARE; else inheritance = VM_INHERIT_DEFAULT; cred = NULL; if ((cow & (MAP_ACC_NO_CHARGE | MAP_NOFAULT | MAP_CREATE_GUARD)) != 0) goto charged; if ((cow & MAP_ACC_CHARGED) || ((prot & VM_PROT_WRITE) && ((protoeflags & MAP_ENTRY_NEEDS_COPY) || object == NULL))) { if (!(cow & MAP_ACC_CHARGED) && !swap_reserve(end - start)) return (KERN_RESOURCE_SHORTAGE); KASSERT(object == NULL || (protoeflags & MAP_ENTRY_NEEDS_COPY) != 0 || object->cred == NULL, ("overcommit: vm_map_insert o %p", object)); cred = curthread->td_ucred; } charged: /* Expand the kernel pmap, if necessary. */ if (map == kernel_map && end > kernel_vm_end) pmap_growkernel(end); if (object != NULL) { /* * OBJ_ONEMAPPING must be cleared unless this mapping * is trivially proven to be the only mapping for any * of the object's pages. (Object granularity * reference counting is insufficient to recognize * aliases with precision.) */ VM_OBJECT_WLOCK(object); if (object->ref_count > 1 || object->shadow_count != 0) vm_object_clear_flag(object, OBJ_ONEMAPPING); VM_OBJECT_WUNLOCK(object); } else if ((prev_entry->eflags & ~MAP_ENTRY_USER_WIRED) == protoeflags && (cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP | MAP_VN_EXEC)) == 0 && prev_entry->end == start && (prev_entry->cred == cred || (prev_entry->object.vm_object != NULL && prev_entry->object.vm_object->cred == cred)) && vm_object_coalesce(prev_entry->object.vm_object, prev_entry->offset, (vm_size_t)(prev_entry->end - prev_entry->start), (vm_size_t)(end - prev_entry->end), cred != NULL && (protoeflags & MAP_ENTRY_NEEDS_COPY) == 0)) { /* * We were able to extend the object. Determine if we * can extend the previous map entry to include the * new range as well. */ if (prev_entry->inheritance == inheritance && prev_entry->protection == prot && prev_entry->max_protection == max && prev_entry->wired_count == 0) { KASSERT((prev_entry->eflags & MAP_ENTRY_USER_WIRED) == 0, ("prev_entry %p has incoherent wiring", prev_entry)); if ((prev_entry->eflags & MAP_ENTRY_GUARD) == 0) map->size += end - prev_entry->end; vm_map_entry_resize(map, prev_entry, end - prev_entry->end); vm_map_try_merge_entries(map, prev_entry, vm_map_entry_succ(prev_entry)); return (KERN_SUCCESS); } /* * If we can extend the object but cannot extend the * map entry, we have to create a new map entry. We * must bump the ref count on the extended object to * account for it. object may be NULL. */ object = prev_entry->object.vm_object; offset = prev_entry->offset + (prev_entry->end - prev_entry->start); vm_object_reference(object); if (cred != NULL && object != NULL && object->cred != NULL && !(prev_entry->eflags & MAP_ENTRY_NEEDS_COPY)) { /* Object already accounts for this uid. */ cred = NULL; } } if (cred != NULL) crhold(cred); /* * Create a new entry */ new_entry = vm_map_entry_create(map); new_entry->start = start; new_entry->end = end; new_entry->cred = NULL; new_entry->eflags = protoeflags; new_entry->object.vm_object = object; new_entry->offset = offset; new_entry->inheritance = inheritance; new_entry->protection = prot; new_entry->max_protection = max; new_entry->wired_count = 0; new_entry->wiring_thread = NULL; new_entry->read_ahead = VM_FAULT_READ_AHEAD_INIT; new_entry->next_read = start; KASSERT(cred == NULL || !ENTRY_CHARGED(new_entry), ("overcommit: vm_map_insert leaks vm_map %p", new_entry)); new_entry->cred = cred; /* * Insert the new entry into the list */ vm_map_entry_link(map, new_entry); if ((new_entry->eflags & MAP_ENTRY_GUARD) == 0) map->size += new_entry->end - new_entry->start; /* * Try to coalesce the new entry with both the previous and next * entries in the list. Previously, we only attempted to coalesce * with the previous entry when object is NULL. Here, we handle the * other cases, which are less common. */ vm_map_try_merge_entries(map, prev_entry, new_entry); vm_map_try_merge_entries(map, new_entry, vm_map_entry_succ(new_entry)); if ((cow & (MAP_PREFAULT | MAP_PREFAULT_PARTIAL)) != 0) { vm_map_pmap_enter(map, start, prot, object, OFF_TO_IDX(offset), end - start, cow & MAP_PREFAULT_PARTIAL); } return (KERN_SUCCESS); } /* * vm_map_findspace: * * Find the first fit (lowest VM address) for "length" free bytes * beginning at address >= start in the given map. * * In a vm_map_entry, "max_free" is the maximum amount of * contiguous free space between an entry in its subtree and a * neighbor of that entry. This allows finding a free region in * one path down the tree, so O(log n) amortized with splay * trees. * * The map must be locked, and leaves it so. * * Returns: starting address if sufficient space, * vm_map_max(map)-length+1 if insufficient space. */ vm_offset_t vm_map_findspace(vm_map_t map, vm_offset_t start, vm_size_t length) { vm_map_entry_t llist, rlist, root, y; vm_size_t left_length; vm_offset_t gap_end; /* * Request must fit within min/max VM address and must avoid * address wrap. */ start = MAX(start, vm_map_min(map)); if (start >= vm_map_max(map) || length > vm_map_max(map) - start) return (vm_map_max(map) - length + 1); /* Empty tree means wide open address space. */ if (map->root == NULL) return (start); /* * After splay_split, if start is within an entry, push it to the start * of the following gap. If rlist is at the end of the gap containing * start, save the end of that gap in gap_end to see if the gap is big * enough; otherwise set gap_end to start skip gap-checking and move * directly to a search of the right subtree. */ root = vm_map_splay_split(map, start, length, &llist, &rlist); gap_end = rlist->start; if (root != NULL) { start = root->end; if (root->right != NULL) gap_end = start; } else if (rlist != &map->header) { root = rlist; rlist = root->left; root->left = NULL; } else { root = llist; llist = root->right; root->right = NULL; } vm_map_splay_merge(map, root, llist, rlist); VM_MAP_ASSERT_CONSISTENT(map); if (length <= gap_end - start) return (start); /* With max_free, can immediately tell if no solution. */ if (root->right == NULL || length > root->right->max_free) return (vm_map_max(map) - length + 1); /* * Splay for the least large-enough gap in the right subtree. */ llist = rlist = &map->header; for (left_length = 0;; left_length = vm_map_entry_max_free_left(root, llist)) { if (length <= left_length) SPLAY_LEFT_STEP(root, y, rlist, length <= vm_map_entry_max_free_left(y, llist)); else SPLAY_RIGHT_STEP(root, y, llist, length > vm_map_entry_max_free_left(y, root)); if (root == NULL) break; } root = llist; llist = root->right; root->right = NULL; if (rlist != &map->header) { y = rlist; rlist = y->left; y->left = NULL; vm_map_splay_merge(map, y, &map->header, rlist); y->max_free = MAX( vm_map_entry_max_free_left(y, root), vm_map_entry_max_free_right(y, &map->header)); root->right = y; } vm_map_splay_merge(map, root, llist, &map->header); VM_MAP_ASSERT_CONSISTENT(map); return (root->end); } int vm_map_fixed(vm_map_t map, vm_object_t object, vm_ooffset_t offset, vm_offset_t start, vm_size_t length, vm_prot_t prot, vm_prot_t max, int cow) { vm_offset_t end; int result; end = start + length; KASSERT((cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) == 0 || object == NULL, ("vm_map_fixed: non-NULL backing object for stack")); vm_map_lock(map); VM_MAP_RANGE_CHECK(map, start, end); if ((cow & MAP_CHECK_EXCL) == 0) vm_map_delete(map, start, end); if ((cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) != 0) { result = vm_map_stack_locked(map, start, length, sgrowsiz, prot, max, cow); } else { result = vm_map_insert(map, object, offset, start, end, prot, max, cow); } vm_map_unlock(map); return (result); } static const int aslr_pages_rnd_64[2] = {0x1000, 0x10}; static const int aslr_pages_rnd_32[2] = {0x100, 0x4}; static int cluster_anon = 1; SYSCTL_INT(_vm, OID_AUTO, cluster_anon, CTLFLAG_RW, &cluster_anon, 0, "Cluster anonymous mappings: 0 = no, 1 = yes if no hint, 2 = always"); static bool clustering_anon_allowed(vm_offset_t addr) { switch (cluster_anon) { case 0: return (false); case 1: return (addr == 0); case 2: default: return (true); } } static long aslr_restarts; SYSCTL_LONG(_vm, OID_AUTO, aslr_restarts, CTLFLAG_RD, &aslr_restarts, 0, "Number of aslr failures"); #define MAP_32BIT_MAX_ADDR ((vm_offset_t)1 << 31) /* * Searches for the specified amount of free space in the given map with the * specified alignment. Performs an address-ordered, first-fit search from * the given address "*addr", with an optional upper bound "max_addr". If the * parameter "alignment" is zero, then the alignment is computed from the * given (object, offset) pair so as to enable the greatest possible use of * superpage mappings. Returns KERN_SUCCESS and the address of the free space * in "*addr" if successful. Otherwise, returns KERN_NO_SPACE. * * The map must be locked. Initially, there must be at least "length" bytes * of free space at the given address. */ static int vm_map_alignspace(vm_map_t map, vm_object_t object, vm_ooffset_t offset, vm_offset_t *addr, vm_size_t length, vm_offset_t max_addr, vm_offset_t alignment) { vm_offset_t aligned_addr, free_addr; VM_MAP_ASSERT_LOCKED(map); free_addr = *addr; KASSERT(free_addr == vm_map_findspace(map, free_addr, length), ("caller failed to provide space %#jx at address %p", (uintmax_t)length, (void *)free_addr)); for (;;) { /* * At the start of every iteration, the free space at address * "*addr" is at least "length" bytes. */ if (alignment == 0) pmap_align_superpage(object, offset, addr, length); else if ((*addr & (alignment - 1)) != 0) { *addr &= ~(alignment - 1); *addr += alignment; } aligned_addr = *addr; if (aligned_addr == free_addr) { /* * Alignment did not change "*addr", so "*addr" must * still provide sufficient free space. */ return (KERN_SUCCESS); } /* * Test for address wrap on "*addr". A wrapped "*addr" could * be a valid address, in which case vm_map_findspace() cannot * be relied upon to fail. */ if (aligned_addr < free_addr) return (KERN_NO_SPACE); *addr = vm_map_findspace(map, aligned_addr, length); if (*addr + length > vm_map_max(map) || (max_addr != 0 && *addr + length > max_addr)) return (KERN_NO_SPACE); free_addr = *addr; if (free_addr == aligned_addr) { /* * If a successful call to vm_map_findspace() did not * change "*addr", then "*addr" must still be aligned * and provide sufficient free space. */ return (KERN_SUCCESS); } } } /* * vm_map_find finds an unallocated region in the target address * map with the given length. The search is defined to be * first-fit from the specified address; the region found is * returned in the same parameter. * * If object is non-NULL, ref count must be bumped by caller * prior to making call to account for the new entry. */ int vm_map_find(vm_map_t map, vm_object_t object, vm_ooffset_t offset, vm_offset_t *addr, /* IN/OUT */ vm_size_t length, vm_offset_t max_addr, int find_space, vm_prot_t prot, vm_prot_t max, int cow) { vm_offset_t alignment, curr_min_addr, min_addr; int gap, pidx, rv, try; bool cluster, en_aslr, update_anon; KASSERT((cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) == 0 || object == NULL, ("vm_map_find: non-NULL backing object for stack")); MPASS((cow & MAP_REMAP) == 0 || (find_space == VMFS_NO_SPACE && (cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) == 0)); if (find_space == VMFS_OPTIMAL_SPACE && (object == NULL || (object->flags & OBJ_COLORED) == 0)) find_space = VMFS_ANY_SPACE; if (find_space >> 8 != 0) { KASSERT((find_space & 0xff) == 0, ("bad VMFS flags")); alignment = (vm_offset_t)1 << (find_space >> 8); } else alignment = 0; en_aslr = (map->flags & MAP_ASLR) != 0; update_anon = cluster = clustering_anon_allowed(*addr) && (map->flags & MAP_IS_SUB_MAP) == 0 && max_addr == 0 && find_space != VMFS_NO_SPACE && object == NULL && (cow & (MAP_INHERIT_SHARE | MAP_STACK_GROWS_UP | MAP_STACK_GROWS_DOWN)) == 0 && prot != PROT_NONE; curr_min_addr = min_addr = *addr; if (en_aslr && min_addr == 0 && !cluster && find_space != VMFS_NO_SPACE && (map->flags & MAP_ASLR_IGNSTART) != 0) curr_min_addr = min_addr = vm_map_min(map); try = 0; vm_map_lock(map); if (cluster) { curr_min_addr = map->anon_loc; if (curr_min_addr == 0) cluster = false; } if (find_space != VMFS_NO_SPACE) { KASSERT(find_space == VMFS_ANY_SPACE || find_space == VMFS_OPTIMAL_SPACE || find_space == VMFS_SUPER_SPACE || alignment != 0, ("unexpected VMFS flag")); again: /* * When creating an anonymous mapping, try clustering * with an existing anonymous mapping first. * * We make up to two attempts to find address space * for a given find_space value. The first attempt may * apply randomization or may cluster with an existing * anonymous mapping. If this first attempt fails, * perform a first-fit search of the available address * space. * * If all tries failed, and find_space is * VMFS_OPTIMAL_SPACE, fallback to VMFS_ANY_SPACE. * Again enable clustering and randomization. */ try++; MPASS(try <= 2); if (try == 2) { /* * Second try: we failed either to find a * suitable region for randomizing the * allocation, or to cluster with an existing * mapping. Retry with free run. */ curr_min_addr = (map->flags & MAP_ASLR_IGNSTART) != 0 ? vm_map_min(map) : min_addr; atomic_add_long(&aslr_restarts, 1); } if (try == 1 && en_aslr && !cluster) { /* * Find space for allocation, including * gap needed for later randomization. */ pidx = MAXPAGESIZES > 1 && pagesizes[1] != 0 && (find_space == VMFS_SUPER_SPACE || find_space == VMFS_OPTIMAL_SPACE) ? 1 : 0; gap = vm_map_max(map) > MAP_32BIT_MAX_ADDR && (max_addr == 0 || max_addr > MAP_32BIT_MAX_ADDR) ? aslr_pages_rnd_64[pidx] : aslr_pages_rnd_32[pidx]; *addr = vm_map_findspace(map, curr_min_addr, length + gap * pagesizes[pidx]); if (*addr + length + gap * pagesizes[pidx] > vm_map_max(map)) goto again; /* And randomize the start address. */ *addr += (arc4random() % gap) * pagesizes[pidx]; if (max_addr != 0 && *addr + length > max_addr) goto again; } else { *addr = vm_map_findspace(map, curr_min_addr, length); if (*addr + length > vm_map_max(map) || (max_addr != 0 && *addr + length > max_addr)) { if (cluster) { cluster = false; MPASS(try == 1); goto again; } rv = KERN_NO_SPACE; goto done; } } if (find_space != VMFS_ANY_SPACE && (rv = vm_map_alignspace(map, object, offset, addr, length, max_addr, alignment)) != KERN_SUCCESS) { if (find_space == VMFS_OPTIMAL_SPACE) { find_space = VMFS_ANY_SPACE; curr_min_addr = min_addr; cluster = update_anon; try = 0; goto again; } goto done; } } else if ((cow & MAP_REMAP) != 0) { if (*addr < vm_map_min(map) || *addr + length > vm_map_max(map) || *addr + length <= length) { rv = KERN_INVALID_ADDRESS; goto done; } vm_map_delete(map, *addr, *addr + length); } if ((cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) != 0) { rv = vm_map_stack_locked(map, *addr, length, sgrowsiz, prot, max, cow); } else { rv = vm_map_insert(map, object, offset, *addr, *addr + length, prot, max, cow); } if (rv == KERN_SUCCESS && update_anon) map->anon_loc = *addr + length; done: vm_map_unlock(map); return (rv); } /* * vm_map_find_min() is a variant of vm_map_find() that takes an * additional parameter (min_addr) and treats the given address * (*addr) differently. Specifically, it treats *addr as a hint * and not as the minimum address where the mapping is created. * * This function works in two phases. First, it tries to * allocate above the hint. If that fails and the hint is * greater than min_addr, it performs a second pass, replacing * the hint with min_addr as the minimum address for the * allocation. */ int vm_map_find_min(vm_map_t map, vm_object_t object, vm_ooffset_t offset, vm_offset_t *addr, vm_size_t length, vm_offset_t min_addr, vm_offset_t max_addr, int find_space, vm_prot_t prot, vm_prot_t max, int cow) { vm_offset_t hint; int rv; hint = *addr; for (;;) { rv = vm_map_find(map, object, offset, addr, length, max_addr, find_space, prot, max, cow); if (rv == KERN_SUCCESS || min_addr >= hint) return (rv); *addr = hint = min_addr; } } /* * A map entry with any of the following flags set must not be merged with * another entry. */ #define MAP_ENTRY_NOMERGE_MASK (MAP_ENTRY_GROWS_DOWN | MAP_ENTRY_GROWS_UP | \ MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_IS_SUB_MAP | MAP_ENTRY_VN_EXEC) static bool vm_map_mergeable_neighbors(vm_map_entry_t prev, vm_map_entry_t entry) { KASSERT((prev->eflags & MAP_ENTRY_NOMERGE_MASK) == 0 || (entry->eflags & MAP_ENTRY_NOMERGE_MASK) == 0, ("vm_map_mergeable_neighbors: neither %p nor %p are mergeable", prev, entry)); return (prev->end == entry->start && prev->object.vm_object == entry->object.vm_object && (prev->object.vm_object == NULL || prev->offset + (prev->end - prev->start) == entry->offset) && prev->eflags == entry->eflags && prev->protection == entry->protection && prev->max_protection == entry->max_protection && prev->inheritance == entry->inheritance && prev->wired_count == entry->wired_count && prev->cred == entry->cred); } static void vm_map_merged_neighbor_dispose(vm_map_t map, vm_map_entry_t entry) { /* * If the backing object is a vnode object, vm_object_deallocate() * calls vrele(). However, vrele() does not lock the vnode because * the vnode has additional references. Thus, the map lock can be * kept without causing a lock-order reversal with the vnode lock. * * Since we count the number of virtual page mappings in * object->un_pager.vnp.writemappings, the writemappings value * should not be adjusted when the entry is disposed of. */ if (entry->object.vm_object != NULL) vm_object_deallocate(entry->object.vm_object); if (entry->cred != NULL) crfree(entry->cred); vm_map_entry_dispose(map, entry); } /* * vm_map_try_merge_entries: * * Compare the given map entry to its predecessor, and merge its precessor * into it if possible. The entry remains valid, and may be extended. * The predecessor may be deleted. * * The map must be locked. */ void vm_map_try_merge_entries(vm_map_t map, vm_map_entry_t prev, vm_map_entry_t entry) { VM_MAP_ASSERT_LOCKED(map); if ((entry->eflags & MAP_ENTRY_NOMERGE_MASK) == 0 && vm_map_mergeable_neighbors(prev, entry)) { vm_map_entry_unlink(map, prev, UNLINK_MERGE_NEXT); vm_map_merged_neighbor_dispose(map, prev); } } /* * vm_map_entry_back: * * Allocate an object to back a map entry. */ static inline void vm_map_entry_back(vm_map_entry_t entry) { vm_object_t object; KASSERT(entry->object.vm_object == NULL, ("map entry %p has backing object", entry)); KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0, ("map entry %p is a submap", entry)); object = vm_object_allocate(OBJT_DEFAULT, atop(entry->end - entry->start)); entry->object.vm_object = object; entry->offset = 0; if (entry->cred != NULL) { object->cred = entry->cred; object->charge = entry->end - entry->start; entry->cred = NULL; } } /* * vm_map_entry_charge_object * * If there is no object backing this entry, create one. Otherwise, if * the entry has cred, give it to the backing object. */ static inline void vm_map_entry_charge_object(vm_map_t map, vm_map_entry_t entry) { VM_MAP_ASSERT_LOCKED(map); KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0, ("map entry %p is a submap", entry)); if (entry->object.vm_object == NULL && !map->system_map && (entry->eflags & MAP_ENTRY_GUARD) == 0) vm_map_entry_back(entry); else if (entry->object.vm_object != NULL && ((entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) && entry->cred != NULL) { VM_OBJECT_WLOCK(entry->object.vm_object); KASSERT(entry->object.vm_object->cred == NULL, ("OVERCOMMIT: %s: both cred e %p", __func__, entry)); entry->object.vm_object->cred = entry->cred; entry->object.vm_object->charge = entry->end - entry->start; VM_OBJECT_WUNLOCK(entry->object.vm_object); entry->cred = NULL; } } /* * vm_map_clip_start: [ internal use only ] * * Asserts that the given entry begins at or after * the specified address; if necessary, * it splits the entry into two. */ #define vm_map_clip_start(map, entry, startaddr) \ { \ if (startaddr > entry->start) \ _vm_map_clip_start(map, entry, startaddr); \ } /* * This routine is called only when it is known that * the entry must be split. */ static void _vm_map_clip_start(vm_map_t map, vm_map_entry_t entry, vm_offset_t start) { vm_map_entry_t new_entry; VM_MAP_ASSERT_LOCKED(map); KASSERT(entry->end > start && entry->start < start, ("_vm_map_clip_start: invalid clip of entry %p", entry)); /* * Create a backing object now, if none exists, so that more individual * objects won't be created after the map entry is split. */ vm_map_entry_charge_object(map, entry); /* Clone the entry. */ new_entry = vm_map_entry_create(map); *new_entry = *entry; /* * Split off the front portion. Insert the new entry BEFORE this one, * so that this entry has the specified starting address. */ new_entry->end = start; entry->offset += (start - entry->start); entry->start = start; if (new_entry->cred != NULL) crhold(entry->cred); vm_map_entry_link(map, new_entry); if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) { vm_object_reference(new_entry->object.vm_object); vm_map_entry_set_vnode_text(new_entry, true); /* * The object->un_pager.vnp.writemappings for the * object of MAP_ENTRY_WRITECNT type entry shall be * kept as is here. The virtual pages are * re-distributed among the clipped entries, so the sum is * left the same. */ } } /* * vm_map_clip_end: [ internal use only ] * * Asserts that the given entry ends at or before * the specified address; if necessary, * it splits the entry into two. */ #define vm_map_clip_end(map, entry, endaddr) \ { \ if ((endaddr) < (entry->end)) \ _vm_map_clip_end((map), (entry), (endaddr)); \ } /* * This routine is called only when it is known that * the entry must be split. */ static void _vm_map_clip_end(vm_map_t map, vm_map_entry_t entry, vm_offset_t end) { vm_map_entry_t new_entry; VM_MAP_ASSERT_LOCKED(map); KASSERT(entry->start < end && entry->end > end, ("_vm_map_clip_end: invalid clip of entry %p", entry)); /* * Create a backing object now, if none exists, so that more individual * objects won't be created after the map entry is split. */ vm_map_entry_charge_object(map, entry); /* Clone the entry. */ new_entry = vm_map_entry_create(map); *new_entry = *entry; /* * Split off the back portion. Insert the new entry AFTER this one, * so that this entry has the specified ending address. */ new_entry->start = entry->end = end; new_entry->offset += (end - entry->start); if (new_entry->cred != NULL) crhold(entry->cred); vm_map_entry_link(map, new_entry); if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) { vm_object_reference(new_entry->object.vm_object); vm_map_entry_set_vnode_text(new_entry, true); } } /* * vm_map_submap: [ kernel use only ] * * Mark the given range as handled by a subordinate map. * * This range must have been created with vm_map_find, * and no other operations may have been performed on this * range prior to calling vm_map_submap. * * Only a limited number of operations can be performed * within this rage after calling vm_map_submap: * vm_fault * [Don't try vm_map_copy!] * * To remove a submapping, one must first remove the * range from the superior map, and then destroy the * submap (if desired). [Better yet, don't try it.] */ int vm_map_submap( vm_map_t map, vm_offset_t start, vm_offset_t end, vm_map_t submap) { vm_map_entry_t entry; int result; result = KERN_INVALID_ARGUMENT; vm_map_lock(submap); submap->flags |= MAP_IS_SUB_MAP; vm_map_unlock(submap); vm_map_lock(map); VM_MAP_RANGE_CHECK(map, start, end); if (vm_map_lookup_entry(map, start, &entry)) { vm_map_clip_start(map, entry, start); } else entry = vm_map_entry_succ(entry); vm_map_clip_end(map, entry, end); if ((entry->start == start) && (entry->end == end) && ((entry->eflags & MAP_ENTRY_COW) == 0) && (entry->object.vm_object == NULL)) { entry->object.sub_map = submap; entry->eflags |= MAP_ENTRY_IS_SUB_MAP; result = KERN_SUCCESS; } vm_map_unlock(map); if (result != KERN_SUCCESS) { vm_map_lock(submap); submap->flags &= ~MAP_IS_SUB_MAP; vm_map_unlock(submap); } return (result); } /* * The maximum number of pages to map if MAP_PREFAULT_PARTIAL is specified */ #define MAX_INIT_PT 96 /* * vm_map_pmap_enter: * * Preload the specified map's pmap with mappings to the specified * object's memory-resident pages. No further physical pages are * allocated, and no further virtual pages are retrieved from secondary * storage. If the specified flags include MAP_PREFAULT_PARTIAL, then a * limited number of page mappings are created at the low-end of the * specified address range. (For this purpose, a superpage mapping * counts as one page mapping.) Otherwise, all resident pages within * the specified address range are mapped. */ static void vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot, vm_object_t object, vm_pindex_t pindex, vm_size_t size, int flags) { vm_offset_t start; vm_page_t p, p_start; vm_pindex_t mask, psize, threshold, tmpidx; if ((prot & (VM_PROT_READ | VM_PROT_EXECUTE)) == 0 || object == NULL) return; VM_OBJECT_RLOCK(object); if (object->type == OBJT_DEVICE || object->type == OBJT_SG) { VM_OBJECT_RUNLOCK(object); VM_OBJECT_WLOCK(object); if (object->type == OBJT_DEVICE || object->type == OBJT_SG) { pmap_object_init_pt(map->pmap, addr, object, pindex, size); VM_OBJECT_WUNLOCK(object); return; } VM_OBJECT_LOCK_DOWNGRADE(object); } psize = atop(size); if (psize + pindex > object->size) { if (object->size < pindex) { VM_OBJECT_RUNLOCK(object); return; } psize = object->size - pindex; } start = 0; p_start = NULL; threshold = MAX_INIT_PT; p = vm_page_find_least(object, pindex); /* * Assert: the variable p is either (1) the page with the * least pindex greater than or equal to the parameter pindex * or (2) NULL. */ for (; p != NULL && (tmpidx = p->pindex - pindex) < psize; p = TAILQ_NEXT(p, listq)) { /* * don't allow an madvise to blow away our really * free pages allocating pv entries. */ if (((flags & MAP_PREFAULT_MADVISE) != 0 && vm_page_count_severe()) || ((flags & MAP_PREFAULT_PARTIAL) != 0 && tmpidx >= threshold)) { psize = tmpidx; break; } if (vm_page_all_valid(p)) { if (p_start == NULL) { start = addr + ptoa(tmpidx); p_start = p; } /* Jump ahead if a superpage mapping is possible. */ if (p->psind > 0 && ((addr + ptoa(tmpidx)) & (pagesizes[p->psind] - 1)) == 0) { mask = atop(pagesizes[p->psind]) - 1; if (tmpidx + mask < psize && vm_page_ps_test(p, PS_ALL_VALID, NULL)) { p += mask; threshold += mask; } } } else if (p_start != NULL) { pmap_enter_object(map->pmap, start, addr + ptoa(tmpidx), p_start, prot); p_start = NULL; } } if (p_start != NULL) pmap_enter_object(map->pmap, start, addr + ptoa(psize), p_start, prot); VM_OBJECT_RUNLOCK(object); } /* * vm_map_protect: * * Sets the protection of the specified address * region in the target map. If "set_max" is * specified, the maximum protection is to be set; * otherwise, only the current protection is affected. */ int vm_map_protect(vm_map_t map, vm_offset_t start, vm_offset_t end, vm_prot_t new_prot, boolean_t set_max) { vm_map_entry_t current, entry, in_tran; vm_object_t obj; struct ucred *cred; vm_prot_t old_prot; int rv; if (start == end) return (KERN_SUCCESS); again: in_tran = NULL; vm_map_lock(map); /* * Ensure that we are not concurrently wiring pages. vm_map_wire() may * need to fault pages into the map and will drop the map lock while * doing so, and the VM object may end up in an inconsistent state if we * update the protection on the map entry in between faults. */ vm_map_wait_busy(map); VM_MAP_RANGE_CHECK(map, start, end); if (!vm_map_lookup_entry(map, start, &entry)) entry = vm_map_entry_succ(entry); /* * Make a first pass to check for protection violations. */ for (current = entry; current->start < end; current = vm_map_entry_succ(current)) { if ((current->eflags & MAP_ENTRY_GUARD) != 0) continue; if (current->eflags & MAP_ENTRY_IS_SUB_MAP) { vm_map_unlock(map); return (KERN_INVALID_ARGUMENT); } if ((new_prot & current->max_protection) != new_prot) { vm_map_unlock(map); return (KERN_PROTECTION_FAILURE); } if ((current->eflags & MAP_ENTRY_IN_TRANSITION) != 0) in_tran = current; } /* * Postpone the operation until all in-transition map entries have * stabilized. An in-transition entry might already have its pages * wired and wired_count incremented, but not yet have its * MAP_ENTRY_USER_WIRED flag set. In which case, we would fail to call * vm_fault_copy_entry() in the final loop below. */ if (in_tran != NULL) { in_tran->eflags |= MAP_ENTRY_NEEDS_WAKEUP; vm_map_unlock_and_wait(map, 0); goto again; } /* * Before changing the protections, try to reserve swap space for any * private (i.e., copy-on-write) mappings that are transitioning from * read-only to read/write access. If a reservation fails, break out * of this loop early and let the next loop simplify the entries, since * some may now be mergeable. */ rv = KERN_SUCCESS; vm_map_clip_start(map, entry, start); for (current = entry; current->start < end; current = vm_map_entry_succ(current)) { vm_map_clip_end(map, current, end); if (set_max || ((new_prot & ~(current->protection)) & VM_PROT_WRITE) == 0 || ENTRY_CHARGED(current) || (current->eflags & MAP_ENTRY_GUARD) != 0) { continue; } cred = curthread->td_ucred; obj = current->object.vm_object; if (obj == NULL || (current->eflags & MAP_ENTRY_NEEDS_COPY)) { if (!swap_reserve(current->end - current->start)) { rv = KERN_RESOURCE_SHORTAGE; end = current->end; break; } crhold(cred); current->cred = cred; continue; } VM_OBJECT_WLOCK(obj); if (obj->type != OBJT_DEFAULT && obj->type != OBJT_SWAP) { VM_OBJECT_WUNLOCK(obj); continue; } /* * Charge for the whole object allocation now, since * we cannot distinguish between non-charged and * charged clipped mapping of the same object later. */ KASSERT(obj->charge == 0, ("vm_map_protect: object %p overcharged (entry %p)", obj, current)); if (!swap_reserve(ptoa(obj->size))) { VM_OBJECT_WUNLOCK(obj); rv = KERN_RESOURCE_SHORTAGE; end = current->end; break; } crhold(cred); obj->cred = cred; obj->charge = ptoa(obj->size); VM_OBJECT_WUNLOCK(obj); } /* * If enough swap space was available, go back and fix up protections. * Otherwise, just simplify entries, since some may have been modified. * [Note that clipping is not necessary the second time.] */ for (current = entry; current->start < end; vm_map_try_merge_entries(map, vm_map_entry_pred(current), current), current = vm_map_entry_succ(current)) { if (rv != KERN_SUCCESS || (current->eflags & MAP_ENTRY_GUARD) != 0) continue; old_prot = current->protection; if (set_max) current->protection = (current->max_protection = new_prot) & old_prot; else current->protection = new_prot; /* * For user wired map entries, the normal lazy evaluation of * write access upgrades through soft page faults is * undesirable. Instead, immediately copy any pages that are * copy-on-write and enable write access in the physical map. */ if ((current->eflags & MAP_ENTRY_USER_WIRED) != 0 && (current->protection & VM_PROT_WRITE) != 0 && (old_prot & VM_PROT_WRITE) == 0) vm_fault_copy_entry(map, map, current, current, NULL); /* * When restricting access, update the physical map. Worry * about copy-on-write here. */ if ((old_prot & ~current->protection) != 0) { #define MASK(entry) (((entry)->eflags & MAP_ENTRY_COW) ? ~VM_PROT_WRITE : \ VM_PROT_ALL) pmap_protect(map->pmap, current->start, current->end, current->protection & MASK(current)); #undef MASK } } vm_map_try_merge_entries(map, vm_map_entry_pred(current), current); vm_map_unlock(map); return (rv); } /* * vm_map_madvise: * * This routine traverses a processes map handling the madvise * system call. Advisories are classified as either those effecting * the vm_map_entry structure, or those effecting the underlying * objects. */ int vm_map_madvise( vm_map_t map, vm_offset_t start, vm_offset_t end, int behav) { vm_map_entry_t current, entry; bool modify_map; /* * Some madvise calls directly modify the vm_map_entry, in which case * we need to use an exclusive lock on the map and we need to perform * various clipping operations. Otherwise we only need a read-lock * on the map. */ switch(behav) { case MADV_NORMAL: case MADV_SEQUENTIAL: case MADV_RANDOM: case MADV_NOSYNC: case MADV_AUTOSYNC: case MADV_NOCORE: case MADV_CORE: if (start == end) return (0); modify_map = true; vm_map_lock(map); break; case MADV_WILLNEED: case MADV_DONTNEED: case MADV_FREE: if (start == end) return (0); modify_map = false; vm_map_lock_read(map); break; default: return (EINVAL); } /* * Locate starting entry and clip if necessary. */ VM_MAP_RANGE_CHECK(map, start, end); if (vm_map_lookup_entry(map, start, &entry)) { if (modify_map) vm_map_clip_start(map, entry, start); } else { entry = vm_map_entry_succ(entry); } if (modify_map) { /* * madvise behaviors that are implemented in the vm_map_entry. * * We clip the vm_map_entry so that behavioral changes are * limited to the specified address range. */ for (current = entry; current->start < end; current = vm_map_entry_succ(current)) { if (current->eflags & MAP_ENTRY_IS_SUB_MAP) continue; vm_map_clip_end(map, current, end); switch (behav) { case MADV_NORMAL: vm_map_entry_set_behavior(current, MAP_ENTRY_BEHAV_NORMAL); break; case MADV_SEQUENTIAL: vm_map_entry_set_behavior(current, MAP_ENTRY_BEHAV_SEQUENTIAL); break; case MADV_RANDOM: vm_map_entry_set_behavior(current, MAP_ENTRY_BEHAV_RANDOM); break; case MADV_NOSYNC: current->eflags |= MAP_ENTRY_NOSYNC; break; case MADV_AUTOSYNC: current->eflags &= ~MAP_ENTRY_NOSYNC; break; case MADV_NOCORE: current->eflags |= MAP_ENTRY_NOCOREDUMP; break; case MADV_CORE: current->eflags &= ~MAP_ENTRY_NOCOREDUMP; break; default: break; } vm_map_try_merge_entries(map, vm_map_entry_pred(current), current); } vm_map_try_merge_entries(map, vm_map_entry_pred(current), current); vm_map_unlock(map); } else { vm_pindex_t pstart, pend; /* * madvise behaviors that are implemented in the underlying * vm_object. * * Since we don't clip the vm_map_entry, we have to clip * the vm_object pindex and count. */ for (current = entry; current->start < end; current = vm_map_entry_succ(current)) { vm_offset_t useEnd, useStart; if (current->eflags & MAP_ENTRY_IS_SUB_MAP) continue; /* * MADV_FREE would otherwise rewind time to * the creation of the shadow object. Because * we hold the VM map read-locked, neither the * entry's object nor the presence of a * backing object can change. */ if (behav == MADV_FREE && current->object.vm_object != NULL && current->object.vm_object->backing_object != NULL) continue; pstart = OFF_TO_IDX(current->offset); pend = pstart + atop(current->end - current->start); useStart = current->start; useEnd = current->end; if (current->start < start) { pstart += atop(start - current->start); useStart = start; } if (current->end > end) { pend -= atop(current->end - end); useEnd = end; } if (pstart >= pend) continue; /* * Perform the pmap_advise() before clearing * PGA_REFERENCED in vm_page_advise(). Otherwise, a * concurrent pmap operation, such as pmap_remove(), * could clear a reference in the pmap and set * PGA_REFERENCED on the page before the pmap_advise() * had completed. Consequently, the page would appear * referenced based upon an old reference that * occurred before this pmap_advise() ran. */ if (behav == MADV_DONTNEED || behav == MADV_FREE) pmap_advise(map->pmap, useStart, useEnd, behav); vm_object_madvise(current->object.vm_object, pstart, pend, behav); /* * Pre-populate paging structures in the * WILLNEED case. For wired entries, the * paging structures are already populated. */ if (behav == MADV_WILLNEED && current->wired_count == 0) { vm_map_pmap_enter(map, useStart, current->protection, current->object.vm_object, pstart, ptoa(pend - pstart), MAP_PREFAULT_MADVISE ); } } vm_map_unlock_read(map); } return (0); } /* * vm_map_inherit: * * Sets the inheritance of the specified address * range in the target map. Inheritance * affects how the map will be shared with * child maps at the time of vmspace_fork. */ int vm_map_inherit(vm_map_t map, vm_offset_t start, vm_offset_t end, vm_inherit_t new_inheritance) { vm_map_entry_t entry; vm_map_entry_t temp_entry; switch (new_inheritance) { case VM_INHERIT_NONE: case VM_INHERIT_COPY: case VM_INHERIT_SHARE: case VM_INHERIT_ZERO: break; default: return (KERN_INVALID_ARGUMENT); } if (start == end) return (KERN_SUCCESS); vm_map_lock(map); VM_MAP_RANGE_CHECK(map, start, end); if (vm_map_lookup_entry(map, start, &temp_entry)) { entry = temp_entry; vm_map_clip_start(map, entry, start); } else entry = vm_map_entry_succ(temp_entry); while (entry->start < end) { vm_map_clip_end(map, entry, end); if ((entry->eflags & MAP_ENTRY_GUARD) == 0 || new_inheritance != VM_INHERIT_ZERO) entry->inheritance = new_inheritance; vm_map_try_merge_entries(map, vm_map_entry_pred(entry), entry); entry = vm_map_entry_succ(entry); } vm_map_try_merge_entries(map, vm_map_entry_pred(entry), entry); vm_map_unlock(map); return (KERN_SUCCESS); } /* * vm_map_entry_in_transition: * * Release the map lock, and sleep until the entry is no longer in * transition. Awake and acquire the map lock. If the map changed while * another held the lock, lookup a possibly-changed entry at or after the * 'start' position of the old entry. */ static vm_map_entry_t vm_map_entry_in_transition(vm_map_t map, vm_offset_t in_start, vm_offset_t *io_end, bool holes_ok, vm_map_entry_t in_entry) { vm_map_entry_t entry; vm_offset_t start; u_int last_timestamp; VM_MAP_ASSERT_LOCKED(map); KASSERT((in_entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0, ("not in-tranition map entry %p", in_entry)); /* * We have not yet clipped the entry. */ start = MAX(in_start, in_entry->start); in_entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP; last_timestamp = map->timestamp; if (vm_map_unlock_and_wait(map, 0)) { /* * Allow interruption of user wiring/unwiring? */ } vm_map_lock(map); if (last_timestamp + 1 == map->timestamp) return (in_entry); /* * Look again for the entry because the map was modified while it was * unlocked. Specifically, the entry may have been clipped, merged, or * deleted. */ if (!vm_map_lookup_entry(map, start, &entry)) { if (!holes_ok) { *io_end = start; return (NULL); } entry = vm_map_entry_succ(entry); } return (entry); } /* * vm_map_unwire: * * Implements both kernel and user unwiring. */ int vm_map_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end, int flags) { vm_map_entry_t entry, first_entry; int rv; bool first_iteration, holes_ok, need_wakeup, user_unwire; if (start == end) return (KERN_SUCCESS); holes_ok = (flags & VM_MAP_WIRE_HOLESOK) != 0; user_unwire = (flags & VM_MAP_WIRE_USER) != 0; vm_map_lock(map); VM_MAP_RANGE_CHECK(map, start, end); if (!vm_map_lookup_entry(map, start, &first_entry)) { if (holes_ok) first_entry = vm_map_entry_succ(first_entry); else { vm_map_unlock(map); return (KERN_INVALID_ADDRESS); } } first_iteration = true; entry = first_entry; rv = KERN_SUCCESS; while (entry->start < end) { if (entry->eflags & MAP_ENTRY_IN_TRANSITION) { /* * We have not yet clipped the entry. */ entry = vm_map_entry_in_transition(map, start, &end, holes_ok, entry); if (entry == NULL) { if (first_iteration) { vm_map_unlock(map); return (KERN_INVALID_ADDRESS); } rv = KERN_INVALID_ADDRESS; break; } first_entry = first_iteration ? entry : NULL; continue; } first_iteration = false; vm_map_clip_start(map, entry, start); vm_map_clip_end(map, entry, end); /* * Mark the entry in case the map lock is released. (See * above.) */ KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 && entry->wiring_thread == NULL, ("owned map entry %p", entry)); entry->eflags |= MAP_ENTRY_IN_TRANSITION; entry->wiring_thread = curthread; /* * Check the map for holes in the specified region. * If holes_ok, skip this check. */ if (!holes_ok && (entry->end < end && vm_map_entry_succ(entry)->start > entry->end)) { end = entry->end; rv = KERN_INVALID_ADDRESS; break; } /* * If system unwiring, require that the entry is system wired. */ if (!user_unwire && vm_map_entry_system_wired_count(entry) == 0) { end = entry->end; rv = KERN_INVALID_ARGUMENT; break; } entry = vm_map_entry_succ(entry); } need_wakeup = false; if (first_entry == NULL && !vm_map_lookup_entry(map, start, &first_entry)) { KASSERT(holes_ok, ("vm_map_unwire: lookup failed")); first_entry = vm_map_entry_succ(first_entry); } for (entry = first_entry; entry->start < end; entry = vm_map_entry_succ(entry)) { /* * If holes_ok was specified, an empty * space in the unwired region could have been mapped * while the map lock was dropped for draining * MAP_ENTRY_IN_TRANSITION. Moreover, another thread * could be simultaneously wiring this new mapping * entry. Detect these cases and skip any entries * marked as in transition by us. */ if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 || entry->wiring_thread != curthread) { KASSERT(holes_ok, ("vm_map_unwire: !HOLESOK and new/changed entry")); continue; } if (rv == KERN_SUCCESS && (!user_unwire || (entry->eflags & MAP_ENTRY_USER_WIRED))) { if (entry->wired_count == 1) vm_map_entry_unwire(map, entry); else entry->wired_count--; if (user_unwire) entry->eflags &= ~MAP_ENTRY_USER_WIRED; } KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0, ("vm_map_unwire: in-transition flag missing %p", entry)); KASSERT(entry->wiring_thread == curthread, ("vm_map_unwire: alien wire %p", entry)); entry->eflags &= ~MAP_ENTRY_IN_TRANSITION; entry->wiring_thread = NULL; if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) { entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP; need_wakeup = true; } vm_map_try_merge_entries(map, vm_map_entry_pred(entry), entry); } vm_map_try_merge_entries(map, vm_map_entry_pred(entry), entry); vm_map_unlock(map); if (need_wakeup) vm_map_wakeup(map); return (rv); } static void vm_map_wire_user_count_sub(u_long npages) { atomic_subtract_long(&vm_user_wire_count, npages); } static bool vm_map_wire_user_count_add(u_long npages) { u_long wired; wired = vm_user_wire_count; do { if (npages + wired > vm_page_max_user_wired) return (false); } while (!atomic_fcmpset_long(&vm_user_wire_count, &wired, npages + wired)); return (true); } /* * vm_map_wire_entry_failure: * * Handle a wiring failure on the given entry. * * The map should be locked. */ static void vm_map_wire_entry_failure(vm_map_t map, vm_map_entry_t entry, vm_offset_t failed_addr) { VM_MAP_ASSERT_LOCKED(map); KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0 && entry->wired_count == 1, ("vm_map_wire_entry_failure: entry %p isn't being wired", entry)); KASSERT(failed_addr < entry->end, ("vm_map_wire_entry_failure: entry %p was fully wired", entry)); /* * If any pages at the start of this entry were successfully wired, * then unwire them. */ if (failed_addr > entry->start) { pmap_unwire(map->pmap, entry->start, failed_addr); vm_object_unwire(entry->object.vm_object, entry->offset, failed_addr - entry->start, PQ_ACTIVE); } /* * Assign an out-of-range value to represent the failure to wire this * entry. */ entry->wired_count = -1; } int vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end, int flags) { int rv; vm_map_lock(map); rv = vm_map_wire_locked(map, start, end, flags); vm_map_unlock(map); return (rv); } /* * vm_map_wire_locked: * * Implements both kernel and user wiring. Returns with the map locked, * the map lock may be dropped. */ int vm_map_wire_locked(vm_map_t map, vm_offset_t start, vm_offset_t end, int flags) { vm_map_entry_t entry, first_entry, tmp_entry; vm_offset_t faddr, saved_end, saved_start; u_long npages; u_int last_timestamp; int rv; bool first_iteration, holes_ok, need_wakeup, user_wire; vm_prot_t prot; VM_MAP_ASSERT_LOCKED(map); if (start == end) return (KERN_SUCCESS); prot = 0; if (flags & VM_MAP_WIRE_WRITE) prot |= VM_PROT_WRITE; holes_ok = (flags & VM_MAP_WIRE_HOLESOK) != 0; user_wire = (flags & VM_MAP_WIRE_USER) != 0; VM_MAP_RANGE_CHECK(map, start, end); if (!vm_map_lookup_entry(map, start, &first_entry)) { if (holes_ok) first_entry = vm_map_entry_succ(first_entry); else return (KERN_INVALID_ADDRESS); } first_iteration = true; entry = first_entry; while (entry->start < end) { if (entry->eflags & MAP_ENTRY_IN_TRANSITION) { /* * We have not yet clipped the entry. */ entry = vm_map_entry_in_transition(map, start, &end, holes_ok, entry); if (entry == NULL) { if (first_iteration) return (KERN_INVALID_ADDRESS); rv = KERN_INVALID_ADDRESS; goto done; } first_entry = first_iteration ? entry : NULL; continue; } first_iteration = false; vm_map_clip_start(map, entry, start); vm_map_clip_end(map, entry, end); /* * Mark the entry in case the map lock is released. (See * above.) */ KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 && entry->wiring_thread == NULL, ("owned map entry %p", entry)); entry->eflags |= MAP_ENTRY_IN_TRANSITION; entry->wiring_thread = curthread; if ((entry->protection & (VM_PROT_READ | VM_PROT_EXECUTE)) == 0 || (entry->protection & prot) != prot) { entry->eflags |= MAP_ENTRY_WIRE_SKIPPED; if (!holes_ok) { end = entry->end; rv = KERN_INVALID_ADDRESS; goto done; } } else if (entry->wired_count == 0) { entry->wired_count++; npages = atop(entry->end - entry->start); if (user_wire && !vm_map_wire_user_count_add(npages)) { vm_map_wire_entry_failure(map, entry, entry->start); end = entry->end; rv = KERN_RESOURCE_SHORTAGE; goto done; } /* * Release the map lock, relying on the in-transition * mark. Mark the map busy for fork. */ saved_start = entry->start; saved_end = entry->end; last_timestamp = map->timestamp; vm_map_busy(map); vm_map_unlock(map); faddr = saved_start; do { /* * Simulate a fault to get the page and enter * it into the physical map. */ if ((rv = vm_fault(map, faddr, VM_PROT_NONE, VM_FAULT_WIRE, NULL)) != KERN_SUCCESS) break; } while ((faddr += PAGE_SIZE) < saved_end); vm_map_lock(map); vm_map_unbusy(map); if (last_timestamp + 1 != map->timestamp) { /* * Look again for the entry because the map was * modified while it was unlocked. The entry * may have been clipped, but NOT merged or * deleted. */ if (!vm_map_lookup_entry(map, saved_start, &tmp_entry)) KASSERT(false, ("vm_map_wire: lookup failed")); if (entry == first_entry) first_entry = tmp_entry; else first_entry = NULL; entry = tmp_entry; while (entry->end < saved_end) { /* * In case of failure, handle entries * that were not fully wired here; * fully wired entries are handled * later. */ if (rv != KERN_SUCCESS && faddr < entry->end) vm_map_wire_entry_failure(map, entry, faddr); entry = vm_map_entry_succ(entry); } } if (rv != KERN_SUCCESS) { vm_map_wire_entry_failure(map, entry, faddr); if (user_wire) vm_map_wire_user_count_sub(npages); end = entry->end; goto done; } } else if (!user_wire || (entry->eflags & MAP_ENTRY_USER_WIRED) == 0) { entry->wired_count++; } /* * Check the map for holes in the specified region. * If holes_ok was specified, skip this check. */ if (!holes_ok && entry->end < end && vm_map_entry_succ(entry)->start > entry->end) { end = entry->end; rv = KERN_INVALID_ADDRESS; goto done; } entry = vm_map_entry_succ(entry); } rv = KERN_SUCCESS; done: need_wakeup = false; if (first_entry == NULL && !vm_map_lookup_entry(map, start, &first_entry)) { KASSERT(holes_ok, ("vm_map_wire: lookup failed")); first_entry = vm_map_entry_succ(first_entry); } for (entry = first_entry; entry->start < end; entry = vm_map_entry_succ(entry)) { /* * If holes_ok was specified, an empty * space in the unwired region could have been mapped * while the map lock was dropped for faulting in the * pages or draining MAP_ENTRY_IN_TRANSITION. * Moreover, another thread could be simultaneously * wiring this new mapping entry. Detect these cases * and skip any entries marked as in transition not by us. */ if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 || entry->wiring_thread != curthread) { KASSERT(holes_ok, ("vm_map_wire: !HOLESOK and new/changed entry")); continue; } if ((entry->eflags & MAP_ENTRY_WIRE_SKIPPED) != 0) { /* do nothing */ } else if (rv == KERN_SUCCESS) { if (user_wire) entry->eflags |= MAP_ENTRY_USER_WIRED; } else if (entry->wired_count == -1) { /* * Wiring failed on this entry. Thus, unwiring is * unnecessary. */ entry->wired_count = 0; } else if (!user_wire || (entry->eflags & MAP_ENTRY_USER_WIRED) == 0) { /* * Undo the wiring. Wiring succeeded on this entry * but failed on a later entry. */ if (entry->wired_count == 1) { vm_map_entry_unwire(map, entry); if (user_wire) vm_map_wire_user_count_sub( atop(entry->end - entry->start)); } else entry->wired_count--; } KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0, ("vm_map_wire: in-transition flag missing %p", entry)); KASSERT(entry->wiring_thread == curthread, ("vm_map_wire: alien wire %p", entry)); entry->eflags &= ~(MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_WIRE_SKIPPED); entry->wiring_thread = NULL; if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) { entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP; need_wakeup = true; } vm_map_try_merge_entries(map, vm_map_entry_pred(entry), entry); } vm_map_try_merge_entries(map, vm_map_entry_pred(entry), entry); if (need_wakeup) vm_map_wakeup(map); return (rv); } /* * vm_map_sync * * Push any dirty cached pages in the address range to their pager. * If syncio is TRUE, dirty pages are written synchronously. * If invalidate is TRUE, any cached pages are freed as well. * * If the size of the region from start to end is zero, we are * supposed to flush all modified pages within the region containing * start. Unfortunately, a region can be split or coalesced with * neighboring regions, making it difficult to determine what the * original region was. Therefore, we approximate this requirement by * flushing the current region containing start. * * Returns an error if any part of the specified range is not mapped. */ int vm_map_sync( vm_map_t map, vm_offset_t start, vm_offset_t end, boolean_t syncio, boolean_t invalidate) { vm_map_entry_t current; vm_map_entry_t entry; vm_size_t size; vm_object_t object; vm_ooffset_t offset; unsigned int last_timestamp; boolean_t failed; vm_map_lock_read(map); VM_MAP_RANGE_CHECK(map, start, end); if (!vm_map_lookup_entry(map, start, &entry)) { vm_map_unlock_read(map); return (KERN_INVALID_ADDRESS); } else if (start == end) { start = entry->start; end = entry->end; } /* * Make a first pass to check for user-wired memory and holes. */ for (current = entry; current->start < end; current = vm_map_entry_succ(current)) { if (invalidate && (current->eflags & MAP_ENTRY_USER_WIRED)) { vm_map_unlock_read(map); return (KERN_INVALID_ARGUMENT); } if (end > current->end && current->end != vm_map_entry_succ(current)->start) { vm_map_unlock_read(map); return (KERN_INVALID_ADDRESS); } } if (invalidate) pmap_remove(map->pmap, start, end); failed = FALSE; /* * Make a second pass, cleaning/uncaching pages from the indicated * objects as we go. */ for (current = entry; current->start < end;) { offset = current->offset + (start - current->start); size = (end <= current->end ? end : current->end) - start; if (current->eflags & MAP_ENTRY_IS_SUB_MAP) { vm_map_t smap; vm_map_entry_t tentry; vm_size_t tsize; smap = current->object.sub_map; vm_map_lock_read(smap); (void) vm_map_lookup_entry(smap, offset, &tentry); tsize = tentry->end - offset; if (tsize < size) size = tsize; object = tentry->object.vm_object; offset = tentry->offset + (offset - tentry->start); vm_map_unlock_read(smap); } else { object = current->object.vm_object; } vm_object_reference(object); last_timestamp = map->timestamp; vm_map_unlock_read(map); if (!vm_object_sync(object, offset, size, syncio, invalidate)) failed = TRUE; start += size; vm_object_deallocate(object); vm_map_lock_read(map); if (last_timestamp == map->timestamp || !vm_map_lookup_entry(map, start, ¤t)) current = vm_map_entry_succ(current); } vm_map_unlock_read(map); return (failed ? KERN_FAILURE : KERN_SUCCESS); } /* * vm_map_entry_unwire: [ internal use only ] * * Make the region specified by this entry pageable. * * The map in question should be locked. * [This is the reason for this routine's existence.] */ static void vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry) { vm_size_t size; VM_MAP_ASSERT_LOCKED(map); KASSERT(entry->wired_count > 0, ("vm_map_entry_unwire: entry %p isn't wired", entry)); size = entry->end - entry->start; if ((entry->eflags & MAP_ENTRY_USER_WIRED) != 0) vm_map_wire_user_count_sub(atop(size)); pmap_unwire(map->pmap, entry->start, entry->end); vm_object_unwire(entry->object.vm_object, entry->offset, size, PQ_ACTIVE); entry->wired_count = 0; } static void vm_map_entry_deallocate(vm_map_entry_t entry, boolean_t system_map) { if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) vm_object_deallocate(entry->object.vm_object); uma_zfree(system_map ? kmapentzone : mapentzone, entry); } /* * vm_map_entry_delete: [ internal use only ] * * Deallocate the given entry from the target map. */ static void vm_map_entry_delete(vm_map_t map, vm_map_entry_t entry) { vm_object_t object; vm_pindex_t offidxstart, offidxend, count, size1; vm_size_t size; vm_map_entry_unlink(map, entry, UNLINK_MERGE_NONE); object = entry->object.vm_object; if ((entry->eflags & MAP_ENTRY_GUARD) != 0) { MPASS(entry->cred == NULL); MPASS((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0); MPASS(object == NULL); vm_map_entry_deallocate(entry, map->system_map); return; } size = entry->end - entry->start; map->size -= size; if (entry->cred != NULL) { swap_release_by_cred(size, entry->cred); crfree(entry->cred); } if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 && (object != NULL)) { KASSERT(entry->cred == NULL || object->cred == NULL || (entry->eflags & MAP_ENTRY_NEEDS_COPY), ("OVERCOMMIT vm_map_entry_delete: both cred %p", entry)); count = atop(size); offidxstart = OFF_TO_IDX(entry->offset); offidxend = offidxstart + count; VM_OBJECT_WLOCK(object); if (object->ref_count != 1 && ((object->flags & (OBJ_NOSPLIT | OBJ_ONEMAPPING)) == OBJ_ONEMAPPING || object == kernel_object)) { vm_object_collapse(object); /* * The option OBJPR_NOTMAPPED can be passed here * because vm_map_delete() already performed * pmap_remove() on the only mapping to this range * of pages. */ vm_object_page_remove(object, offidxstart, offidxend, OBJPR_NOTMAPPED); if (object->type == OBJT_SWAP) swap_pager_freespace(object, offidxstart, count); if (offidxend >= object->size && offidxstart < object->size) { size1 = object->size; object->size = offidxstart; if (object->cred != NULL) { size1 -= object->size; KASSERT(object->charge >= ptoa(size1), ("object %p charge < 0", object)); swap_release_by_cred(ptoa(size1), object->cred); object->charge -= ptoa(size1); } } } VM_OBJECT_WUNLOCK(object); } else entry->object.vm_object = NULL; if (map->system_map) vm_map_entry_deallocate(entry, TRUE); else { entry->defer_next = curthread->td_map_def_user; curthread->td_map_def_user = entry; } } /* * vm_map_delete: [ internal use only ] * * Deallocates the given address range from the target * map. */ int vm_map_delete(vm_map_t map, vm_offset_t start, vm_offset_t end) { vm_map_entry_t entry; vm_map_entry_t first_entry; VM_MAP_ASSERT_LOCKED(map); if (start == end) return (KERN_SUCCESS); /* * Find the start of the region, and clip it */ if (!vm_map_lookup_entry(map, start, &first_entry)) entry = vm_map_entry_succ(first_entry); else { entry = first_entry; vm_map_clip_start(map, entry, start); } /* * Step through all entries in this region */ while (entry->start < end) { vm_map_entry_t next; /* * Wait for wiring or unwiring of an entry to complete. * Also wait for any system wirings to disappear on * user maps. */ if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0 || (vm_map_pmap(map) != kernel_pmap && vm_map_entry_system_wired_count(entry) != 0)) { unsigned int last_timestamp; vm_offset_t saved_start; vm_map_entry_t tmp_entry; saved_start = entry->start; entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP; last_timestamp = map->timestamp; (void) vm_map_unlock_and_wait(map, 0); vm_map_lock(map); if (last_timestamp + 1 != map->timestamp) { /* * Look again for the entry because the map was * modified while it was unlocked. * Specifically, the entry may have been * clipped, merged, or deleted. */ if (!vm_map_lookup_entry(map, saved_start, &tmp_entry)) entry = vm_map_entry_succ(tmp_entry); else { entry = tmp_entry; vm_map_clip_start(map, entry, saved_start); } } continue; } vm_map_clip_end(map, entry, end); next = vm_map_entry_succ(entry); /* * Unwire before removing addresses from the pmap; otherwise, * unwiring will put the entries back in the pmap. */ if (entry->wired_count != 0) vm_map_entry_unwire(map, entry); /* * Remove mappings for the pages, but only if the * mappings could exist. For instance, it does not * make sense to call pmap_remove() for guard entries. */ if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0 || entry->object.vm_object != NULL) pmap_remove(map->pmap, entry->start, entry->end); if (entry->end == map->anon_loc) map->anon_loc = entry->start; /* * Delete the entry only after removing all pmap * entries pointing to its pages. (Otherwise, its * page frames may be reallocated, and any modify bits * will be set in the wrong object!) */ vm_map_entry_delete(map, entry); entry = next; } return (KERN_SUCCESS); } /* * vm_map_remove: * * Remove the given address range from the target map. * This is the exported form of vm_map_delete. */ int vm_map_remove(vm_map_t map, vm_offset_t start, vm_offset_t end) { int result; vm_map_lock(map); VM_MAP_RANGE_CHECK(map, start, end); result = vm_map_delete(map, start, end); vm_map_unlock(map); return (result); } /* * vm_map_check_protection: * * Assert that the target map allows the specified privilege on the * entire address region given. The entire region must be allocated. * * WARNING! This code does not and should not check whether the * contents of the region is accessible. For example a smaller file * might be mapped into a larger address space. * * NOTE! This code is also called by munmap(). * * The map must be locked. A read lock is sufficient. */ boolean_t vm_map_check_protection(vm_map_t map, vm_offset_t start, vm_offset_t end, vm_prot_t protection) { vm_map_entry_t entry; vm_map_entry_t tmp_entry; if (!vm_map_lookup_entry(map, start, &tmp_entry)) return (FALSE); entry = tmp_entry; while (start < end) { /* * No holes allowed! */ if (start < entry->start) return (FALSE); /* * Check protection associated with entry. */ if ((entry->protection & protection) != protection) return (FALSE); /* go to next entry */ start = entry->end; entry = vm_map_entry_succ(entry); } return (TRUE); } /* * vm_map_copy_entry: * * Copies the contents of the source entry to the destination * entry. The entries *must* be aligned properly. */ static void vm_map_copy_entry( vm_map_t src_map, vm_map_t dst_map, vm_map_entry_t src_entry, vm_map_entry_t dst_entry, vm_ooffset_t *fork_charge) { vm_object_t src_object; vm_map_entry_t fake_entry; vm_offset_t size; struct ucred *cred; int charged; VM_MAP_ASSERT_LOCKED(dst_map); if ((dst_entry->eflags|src_entry->eflags) & MAP_ENTRY_IS_SUB_MAP) return; if (src_entry->wired_count == 0 || (src_entry->protection & VM_PROT_WRITE) == 0) { /* * If the source entry is marked needs_copy, it is already * write-protected. */ if ((src_entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0 && (src_entry->protection & VM_PROT_WRITE) != 0) { pmap_protect(src_map->pmap, src_entry->start, src_entry->end, src_entry->protection & ~VM_PROT_WRITE); } /* * Make a copy of the object. */ size = src_entry->end - src_entry->start; if ((src_object = src_entry->object.vm_object) != NULL) { VM_OBJECT_WLOCK(src_object); charged = ENTRY_CHARGED(src_entry); if (src_object->handle == NULL && (src_object->type == OBJT_DEFAULT || src_object->type == OBJT_SWAP)) { vm_object_collapse(src_object); if ((src_object->flags & (OBJ_NOSPLIT | OBJ_ONEMAPPING)) == OBJ_ONEMAPPING) { vm_object_split(src_entry); src_object = src_entry->object.vm_object; } } vm_object_reference_locked(src_object); vm_object_clear_flag(src_object, OBJ_ONEMAPPING); if (src_entry->cred != NULL && !(src_entry->eflags & MAP_ENTRY_NEEDS_COPY)) { KASSERT(src_object->cred == NULL, ("OVERCOMMIT: vm_map_copy_entry: cred %p", src_object)); src_object->cred = src_entry->cred; src_object->charge = size; } VM_OBJECT_WUNLOCK(src_object); dst_entry->object.vm_object = src_object; if (charged) { cred = curthread->td_ucred; crhold(cred); dst_entry->cred = cred; *fork_charge += size; if (!(src_entry->eflags & MAP_ENTRY_NEEDS_COPY)) { crhold(cred); src_entry->cred = cred; *fork_charge += size; } } src_entry->eflags |= MAP_ENTRY_COW | MAP_ENTRY_NEEDS_COPY; dst_entry->eflags |= MAP_ENTRY_COW | MAP_ENTRY_NEEDS_COPY; dst_entry->offset = src_entry->offset; if (src_entry->eflags & MAP_ENTRY_WRITECNT) { /* * MAP_ENTRY_WRITECNT cannot * indicate write reference from * src_entry, since the entry is * marked as needs copy. Allocate a * fake entry that is used to * decrement object->un_pager writecount * at the appropriate time. Attach * fake_entry to the deferred list. */ fake_entry = vm_map_entry_create(dst_map); fake_entry->eflags = MAP_ENTRY_WRITECNT; src_entry->eflags &= ~MAP_ENTRY_WRITECNT; vm_object_reference(src_object); fake_entry->object.vm_object = src_object; fake_entry->start = src_entry->start; fake_entry->end = src_entry->end; fake_entry->defer_next = curthread->td_map_def_user; curthread->td_map_def_user = fake_entry; } pmap_copy(dst_map->pmap, src_map->pmap, dst_entry->start, dst_entry->end - dst_entry->start, src_entry->start); } else { dst_entry->object.vm_object = NULL; dst_entry->offset = 0; if (src_entry->cred != NULL) { dst_entry->cred = curthread->td_ucred; crhold(dst_entry->cred); *fork_charge += size; } } } else { /* * We don't want to make writeable wired pages copy-on-write. * Immediately copy these pages into the new map by simulating * page faults. The new pages are pageable. */ vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry, fork_charge); } } /* * vmspace_map_entry_forked: * Update the newly-forked vmspace each time a map entry is inherited * or copied. The values for vm_dsize and vm_tsize are approximate * (and mostly-obsolete ideas in the face of mmap(2) et al.) */ static void vmspace_map_entry_forked(const struct vmspace *vm1, struct vmspace *vm2, vm_map_entry_t entry) { vm_size_t entrysize; vm_offset_t newend; if ((entry->eflags & MAP_ENTRY_GUARD) != 0) return; entrysize = entry->end - entry->start; vm2->vm_map.size += entrysize; if (entry->eflags & (MAP_ENTRY_GROWS_DOWN | MAP_ENTRY_GROWS_UP)) { vm2->vm_ssize += btoc(entrysize); } else if (entry->start >= (vm_offset_t)vm1->vm_daddr && entry->start < (vm_offset_t)vm1->vm_daddr + ctob(vm1->vm_dsize)) { newend = MIN(entry->end, (vm_offset_t)vm1->vm_daddr + ctob(vm1->vm_dsize)); vm2->vm_dsize += btoc(newend - entry->start); } else if (entry->start >= (vm_offset_t)vm1->vm_taddr && entry->start < (vm_offset_t)vm1->vm_taddr + ctob(vm1->vm_tsize)) { newend = MIN(entry->end, (vm_offset_t)vm1->vm_taddr + ctob(vm1->vm_tsize)); vm2->vm_tsize += btoc(newend - entry->start); } } /* * vmspace_fork: * Create a new process vmspace structure and vm_map * based on those of an existing process. The new map * is based on the old map, according to the inheritance * values on the regions in that map. * * XXX It might be worth coalescing the entries added to the new vmspace. * * The source map must not be locked. */ struct vmspace * vmspace_fork(struct vmspace *vm1, vm_ooffset_t *fork_charge) { struct vmspace *vm2; vm_map_t new_map, old_map; vm_map_entry_t new_entry, old_entry; vm_object_t object; int error, locked; vm_inherit_t inh; old_map = &vm1->vm_map; /* Copy immutable fields of vm1 to vm2. */ vm2 = vmspace_alloc(vm_map_min(old_map), vm_map_max(old_map), pmap_pinit); if (vm2 == NULL) return (NULL); vm2->vm_taddr = vm1->vm_taddr; vm2->vm_daddr = vm1->vm_daddr; vm2->vm_maxsaddr = vm1->vm_maxsaddr; vm_map_lock(old_map); if (old_map->busy) vm_map_wait_busy(old_map); new_map = &vm2->vm_map; locked = vm_map_trylock(new_map); /* trylock to silence WITNESS */ KASSERT(locked, ("vmspace_fork: lock failed")); error = pmap_vmspace_copy(new_map->pmap, old_map->pmap); if (error != 0) { sx_xunlock(&old_map->lock); sx_xunlock(&new_map->lock); vm_map_process_deferred(); vmspace_free(vm2); return (NULL); } new_map->anon_loc = old_map->anon_loc; old_entry = old_map->header.next; while (old_entry != &old_map->header) { if (old_entry->eflags & MAP_ENTRY_IS_SUB_MAP) panic("vm_map_fork: encountered a submap"); inh = old_entry->inheritance; if ((old_entry->eflags & MAP_ENTRY_GUARD) != 0 && inh != VM_INHERIT_NONE) inh = VM_INHERIT_COPY; switch (inh) { case VM_INHERIT_NONE: break; case VM_INHERIT_SHARE: /* * Clone the entry, creating the shared object if necessary. */ object = old_entry->object.vm_object; if (object == NULL) { vm_map_entry_back(old_entry); object = old_entry->object.vm_object; } /* * Add the reference before calling vm_object_shadow * to insure that a shadow object is created. */ vm_object_reference(object); if (old_entry->eflags & MAP_ENTRY_NEEDS_COPY) { vm_object_shadow(&old_entry->object.vm_object, &old_entry->offset, old_entry->end - old_entry->start); old_entry->eflags &= ~MAP_ENTRY_NEEDS_COPY; /* Transfer the second reference too. */ vm_object_reference( old_entry->object.vm_object); /* * As in vm_map_merged_neighbor_dispose(), * the vnode lock will not be acquired in * this call to vm_object_deallocate(). */ vm_object_deallocate(object); object = old_entry->object.vm_object; } VM_OBJECT_WLOCK(object); vm_object_clear_flag(object, OBJ_ONEMAPPING); if (old_entry->cred != NULL) { KASSERT(object->cred == NULL, ("vmspace_fork both cred")); object->cred = old_entry->cred; object->charge = old_entry->end - old_entry->start; old_entry->cred = NULL; } /* * Assert the correct state of the vnode * v_writecount while the object is locked, to * not relock it later for the assertion * correctness. */ if (old_entry->eflags & MAP_ENTRY_WRITECNT && object->type == OBJT_VNODE) { KASSERT(((struct vnode *)object->handle)-> v_writecount > 0, ("vmspace_fork: v_writecount %p", object)); KASSERT(object->un_pager.vnp.writemappings > 0, ("vmspace_fork: vnp.writecount %p", object)); } VM_OBJECT_WUNLOCK(object); /* * Clone the entry, referencing the shared object. */ new_entry = vm_map_entry_create(new_map); *new_entry = *old_entry; new_entry->eflags &= ~(MAP_ENTRY_USER_WIRED | MAP_ENTRY_IN_TRANSITION); new_entry->wiring_thread = NULL; new_entry->wired_count = 0; if (new_entry->eflags & MAP_ENTRY_WRITECNT) { vm_pager_update_writecount(object, new_entry->start, new_entry->end); } vm_map_entry_set_vnode_text(new_entry, true); /* * Insert the entry into the new map -- we know we're * inserting at the end of the new map. */ vm_map_entry_link(new_map, new_entry); vmspace_map_entry_forked(vm1, vm2, new_entry); /* * Update the physical map */ pmap_copy(new_map->pmap, old_map->pmap, new_entry->start, (old_entry->end - old_entry->start), old_entry->start); break; case VM_INHERIT_COPY: /* * Clone the entry and link into the map. */ new_entry = vm_map_entry_create(new_map); *new_entry = *old_entry; /* * Copied entry is COW over the old object. */ new_entry->eflags &= ~(MAP_ENTRY_USER_WIRED | MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_WRITECNT); new_entry->wiring_thread = NULL; new_entry->wired_count = 0; new_entry->object.vm_object = NULL; new_entry->cred = NULL; vm_map_entry_link(new_map, new_entry); vmspace_map_entry_forked(vm1, vm2, new_entry); vm_map_copy_entry(old_map, new_map, old_entry, new_entry, fork_charge); vm_map_entry_set_vnode_text(new_entry, true); break; case VM_INHERIT_ZERO: /* * Create a new anonymous mapping entry modelled from * the old one. */ new_entry = vm_map_entry_create(new_map); memset(new_entry, 0, sizeof(*new_entry)); new_entry->start = old_entry->start; new_entry->end = old_entry->end; new_entry->eflags = old_entry->eflags & ~(MAP_ENTRY_USER_WIRED | MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_WRITECNT | MAP_ENTRY_VN_EXEC); new_entry->protection = old_entry->protection; new_entry->max_protection = old_entry->max_protection; new_entry->inheritance = VM_INHERIT_ZERO; vm_map_entry_link(new_map, new_entry); vmspace_map_entry_forked(vm1, vm2, new_entry); new_entry->cred = curthread->td_ucred; crhold(new_entry->cred); *fork_charge += (new_entry->end - new_entry->start); break; } old_entry = vm_map_entry_succ(old_entry); } /* * Use inlined vm_map_unlock() to postpone handling the deferred * map entries, which cannot be done until both old_map and * new_map locks are released. */ sx_xunlock(&old_map->lock); sx_xunlock(&new_map->lock); vm_map_process_deferred(); return (vm2); } /* * Create a process's stack for exec_new_vmspace(). This function is never * asked to wire the newly created stack. */ int vm_map_stack(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize, vm_prot_t prot, vm_prot_t max, int cow) { vm_size_t growsize, init_ssize; rlim_t vmemlim; int rv; MPASS((map->flags & MAP_WIREFUTURE) == 0); growsize = sgrowsiz; init_ssize = (max_ssize < growsize) ? max_ssize : growsize; vm_map_lock(map); vmemlim = lim_cur(curthread, RLIMIT_VMEM); /* If we would blow our VMEM resource limit, no go */ if (map->size + init_ssize > vmemlim) { rv = KERN_NO_SPACE; goto out; } rv = vm_map_stack_locked(map, addrbos, max_ssize, growsize, prot, max, cow); out: vm_map_unlock(map); return (rv); } static int stack_guard_page = 1; SYSCTL_INT(_security_bsd, OID_AUTO, stack_guard_page, CTLFLAG_RWTUN, &stack_guard_page, 0, "Specifies the number of guard pages for a stack that grows"); static int vm_map_stack_locked(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize, vm_size_t growsize, vm_prot_t prot, vm_prot_t max, int cow) { vm_map_entry_t new_entry, prev_entry; vm_offset_t bot, gap_bot, gap_top, top; vm_size_t init_ssize, sgp; int orient, rv; /* * The stack orientation is piggybacked with the cow argument. * Extract it into orient and mask the cow argument so that we * don't pass it around further. */ orient = cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP); KASSERT(orient != 0, ("No stack grow direction")); KASSERT(orient != (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP), ("bi-dir stack")); if (addrbos < vm_map_min(map) || addrbos + max_ssize > vm_map_max(map) || addrbos + max_ssize <= addrbos) return (KERN_INVALID_ADDRESS); - sgp = (curproc->p_flag2 & P2_STKGAP_DISABLE) != 0 ? 0 : + sgp = ((curproc->p_flag2 & P2_STKGAP_DISABLE) != 0 || + (curproc->p_fctl0 & NT_FREEBSD_FCTL_STKGAP_DISABLE) != 0) ? 0 : (vm_size_t)stack_guard_page * PAGE_SIZE; if (sgp >= max_ssize) return (KERN_INVALID_ARGUMENT); init_ssize = growsize; if (max_ssize < init_ssize + sgp) init_ssize = max_ssize - sgp; /* If addr is already mapped, no go */ if (vm_map_lookup_entry(map, addrbos, &prev_entry)) return (KERN_NO_SPACE); /* * If we can't accommodate max_ssize in the current mapping, no go. */ if (vm_map_entry_succ(prev_entry)->start < addrbos + max_ssize) return (KERN_NO_SPACE); /* * We initially map a stack of only init_ssize. We will grow as * needed later. Depending on the orientation of the stack (i.e. * the grow direction) we either map at the top of the range, the * bottom of the range or in the middle. * * Note: we would normally expect prot and max to be VM_PROT_ALL, * and cow to be 0. Possibly we should eliminate these as input * parameters, and just pass these values here in the insert call. */ if (orient == MAP_STACK_GROWS_DOWN) { bot = addrbos + max_ssize - init_ssize; top = bot + init_ssize; gap_bot = addrbos; gap_top = bot; } else /* if (orient == MAP_STACK_GROWS_UP) */ { bot = addrbos; top = bot + init_ssize; gap_bot = top; gap_top = addrbos + max_ssize; } rv = vm_map_insert(map, NULL, 0, bot, top, prot, max, cow); if (rv != KERN_SUCCESS) return (rv); new_entry = vm_map_entry_succ(prev_entry); KASSERT(new_entry->end == top || new_entry->start == bot, ("Bad entry start/end for new stack entry")); KASSERT((orient & MAP_STACK_GROWS_DOWN) == 0 || (new_entry->eflags & MAP_ENTRY_GROWS_DOWN) != 0, ("new entry lacks MAP_ENTRY_GROWS_DOWN")); KASSERT((orient & MAP_STACK_GROWS_UP) == 0 || (new_entry->eflags & MAP_ENTRY_GROWS_UP) != 0, ("new entry lacks MAP_ENTRY_GROWS_UP")); if (gap_bot == gap_top) return (KERN_SUCCESS); rv = vm_map_insert(map, NULL, 0, gap_bot, gap_top, VM_PROT_NONE, VM_PROT_NONE, MAP_CREATE_GUARD | (orient == MAP_STACK_GROWS_DOWN ? MAP_CREATE_STACK_GAP_DN : MAP_CREATE_STACK_GAP_UP)); if (rv == KERN_SUCCESS) { /* * Gap can never successfully handle a fault, so * read-ahead logic is never used for it. Re-use * next_read of the gap entry to store * stack_guard_page for vm_map_growstack(). */ if (orient == MAP_STACK_GROWS_DOWN) vm_map_entry_pred(new_entry)->next_read = sgp; else vm_map_entry_succ(new_entry)->next_read = sgp; } else { (void)vm_map_delete(map, bot, top); } return (rv); } /* * Attempts to grow a vm stack entry. Returns KERN_SUCCESS if we * successfully grow the stack. */ static int vm_map_growstack(vm_map_t map, vm_offset_t addr, vm_map_entry_t gap_entry) { vm_map_entry_t stack_entry; struct proc *p; struct vmspace *vm; struct ucred *cred; vm_offset_t gap_end, gap_start, grow_start; vm_size_t grow_amount, guard, max_grow; rlim_t lmemlim, stacklim, vmemlim; int rv, rv1; bool gap_deleted, grow_down, is_procstack; #ifdef notyet uint64_t limit; #endif #ifdef RACCT int error; #endif p = curproc; vm = p->p_vmspace; /* * Disallow stack growth when the access is performed by a * debugger or AIO daemon. The reason is that the wrong * resource limits are applied. */ if (p != initproc && (map != &p->p_vmspace->vm_map || p->p_textvp == NULL)) return (KERN_FAILURE); MPASS(!map->system_map); lmemlim = lim_cur(curthread, RLIMIT_MEMLOCK); stacklim = lim_cur(curthread, RLIMIT_STACK); vmemlim = lim_cur(curthread, RLIMIT_VMEM); retry: /* If addr is not in a hole for a stack grow area, no need to grow. */ if (gap_entry == NULL && !vm_map_lookup_entry(map, addr, &gap_entry)) return (KERN_FAILURE); if ((gap_entry->eflags & MAP_ENTRY_GUARD) == 0) return (KERN_SUCCESS); if ((gap_entry->eflags & MAP_ENTRY_STACK_GAP_DN) != 0) { stack_entry = vm_map_entry_succ(gap_entry); if ((stack_entry->eflags & MAP_ENTRY_GROWS_DOWN) == 0 || stack_entry->start != gap_entry->end) return (KERN_FAILURE); grow_amount = round_page(stack_entry->start - addr); grow_down = true; } else if ((gap_entry->eflags & MAP_ENTRY_STACK_GAP_UP) != 0) { stack_entry = vm_map_entry_pred(gap_entry); if ((stack_entry->eflags & MAP_ENTRY_GROWS_UP) == 0 || stack_entry->end != gap_entry->start) return (KERN_FAILURE); grow_amount = round_page(addr + 1 - stack_entry->end); grow_down = false; } else { return (KERN_FAILURE); } - guard = (curproc->p_flag2 & P2_STKGAP_DISABLE) != 0 ? 0 : + guard = ((curproc->p_flag2 & P2_STKGAP_DISABLE) != 0 || + (curproc->p_fctl0 & NT_FREEBSD_FCTL_STKGAP_DISABLE) != 0) ? 0 : gap_entry->next_read; max_grow = gap_entry->end - gap_entry->start; if (guard > max_grow) return (KERN_NO_SPACE); max_grow -= guard; if (grow_amount > max_grow) return (KERN_NO_SPACE); /* * If this is the main process stack, see if we're over the stack * limit. */ is_procstack = addr >= (vm_offset_t)vm->vm_maxsaddr && addr < (vm_offset_t)p->p_sysent->sv_usrstack; if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > stacklim)) return (KERN_NO_SPACE); #ifdef RACCT if (racct_enable) { PROC_LOCK(p); if (is_procstack && racct_set(p, RACCT_STACK, ctob(vm->vm_ssize) + grow_amount)) { PROC_UNLOCK(p); return (KERN_NO_SPACE); } PROC_UNLOCK(p); } #endif grow_amount = roundup(grow_amount, sgrowsiz); if (grow_amount > max_grow) grow_amount = max_grow; if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > stacklim)) { grow_amount = trunc_page((vm_size_t)stacklim) - ctob(vm->vm_ssize); } #ifdef notyet PROC_LOCK(p); limit = racct_get_available(p, RACCT_STACK); PROC_UNLOCK(p); if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > limit)) grow_amount = limit - ctob(vm->vm_ssize); #endif if (!old_mlock && (map->flags & MAP_WIREFUTURE) != 0) { if (ptoa(pmap_wired_count(map->pmap)) + grow_amount > lmemlim) { rv = KERN_NO_SPACE; goto out; } #ifdef RACCT if (racct_enable) { PROC_LOCK(p); if (racct_set(p, RACCT_MEMLOCK, ptoa(pmap_wired_count(map->pmap)) + grow_amount)) { PROC_UNLOCK(p); rv = KERN_NO_SPACE; goto out; } PROC_UNLOCK(p); } #endif } /* If we would blow our VMEM resource limit, no go */ if (map->size + grow_amount > vmemlim) { rv = KERN_NO_SPACE; goto out; } #ifdef RACCT if (racct_enable) { PROC_LOCK(p); if (racct_set(p, RACCT_VMEM, map->size + grow_amount)) { PROC_UNLOCK(p); rv = KERN_NO_SPACE; goto out; } PROC_UNLOCK(p); } #endif if (vm_map_lock_upgrade(map)) { gap_entry = NULL; vm_map_lock_read(map); goto retry; } if (grow_down) { grow_start = gap_entry->end - grow_amount; if (gap_entry->start + grow_amount == gap_entry->end) { gap_start = gap_entry->start; gap_end = gap_entry->end; vm_map_entry_delete(map, gap_entry); gap_deleted = true; } else { MPASS(gap_entry->start < gap_entry->end - grow_amount); vm_map_entry_resize(map, gap_entry, -grow_amount); gap_deleted = false; } rv = vm_map_insert(map, NULL, 0, grow_start, grow_start + grow_amount, stack_entry->protection, stack_entry->max_protection, MAP_STACK_GROWS_DOWN); if (rv != KERN_SUCCESS) { if (gap_deleted) { rv1 = vm_map_insert(map, NULL, 0, gap_start, gap_end, VM_PROT_NONE, VM_PROT_NONE, MAP_CREATE_GUARD | MAP_CREATE_STACK_GAP_DN); MPASS(rv1 == KERN_SUCCESS); } else vm_map_entry_resize(map, gap_entry, grow_amount); } } else { grow_start = stack_entry->end; cred = stack_entry->cred; if (cred == NULL && stack_entry->object.vm_object != NULL) cred = stack_entry->object.vm_object->cred; if (cred != NULL && !swap_reserve_by_cred(grow_amount, cred)) rv = KERN_NO_SPACE; /* Grow the underlying object if applicable. */ else if (stack_entry->object.vm_object == NULL || vm_object_coalesce(stack_entry->object.vm_object, stack_entry->offset, (vm_size_t)(stack_entry->end - stack_entry->start), grow_amount, cred != NULL)) { if (gap_entry->start + grow_amount == gap_entry->end) { vm_map_entry_delete(map, gap_entry); vm_map_entry_resize(map, stack_entry, grow_amount); } else { gap_entry->start += grow_amount; stack_entry->end += grow_amount; } map->size += grow_amount; rv = KERN_SUCCESS; } else rv = KERN_FAILURE; } if (rv == KERN_SUCCESS && is_procstack) vm->vm_ssize += btoc(grow_amount); /* * Heed the MAP_WIREFUTURE flag if it was set for this process. */ if (rv == KERN_SUCCESS && (map->flags & MAP_WIREFUTURE) != 0) { rv = vm_map_wire_locked(map, grow_start, grow_start + grow_amount, VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); } vm_map_lock_downgrade(map); out: #ifdef RACCT if (racct_enable && rv != KERN_SUCCESS) { PROC_LOCK(p); error = racct_set(p, RACCT_VMEM, map->size); KASSERT(error == 0, ("decreasing RACCT_VMEM failed")); if (!old_mlock) { error = racct_set(p, RACCT_MEMLOCK, ptoa(pmap_wired_count(map->pmap))); KASSERT(error == 0, ("decreasing RACCT_MEMLOCK failed")); } error = racct_set(p, RACCT_STACK, ctob(vm->vm_ssize)); KASSERT(error == 0, ("decreasing RACCT_STACK failed")); PROC_UNLOCK(p); } #endif return (rv); } /* * Unshare the specified VM space for exec. If other processes are * mapped to it, then create a new one. The new vmspace is null. */ int vmspace_exec(struct proc *p, vm_offset_t minuser, vm_offset_t maxuser) { struct vmspace *oldvmspace = p->p_vmspace; struct vmspace *newvmspace; KASSERT((curthread->td_pflags & TDP_EXECVMSPC) == 0, ("vmspace_exec recursed")); newvmspace = vmspace_alloc(minuser, maxuser, pmap_pinit); if (newvmspace == NULL) return (ENOMEM); newvmspace->vm_swrss = oldvmspace->vm_swrss; /* * This code is written like this for prototype purposes. The * goal is to avoid running down the vmspace here, but let the * other process's that are still using the vmspace to finally * run it down. Even though there is little or no chance of blocking * here, it is a good idea to keep this form for future mods. */ PROC_VMSPACE_LOCK(p); p->p_vmspace = newvmspace; PROC_VMSPACE_UNLOCK(p); if (p == curthread->td_proc) pmap_activate(curthread); curthread->td_pflags |= TDP_EXECVMSPC; return (0); } /* * Unshare the specified VM space for forcing COW. This * is called by rfork, for the (RFMEM|RFPROC) == 0 case. */ int vmspace_unshare(struct proc *p) { struct vmspace *oldvmspace = p->p_vmspace; struct vmspace *newvmspace; vm_ooffset_t fork_charge; if (oldvmspace->vm_refcnt == 1) return (0); fork_charge = 0; newvmspace = vmspace_fork(oldvmspace, &fork_charge); if (newvmspace == NULL) return (ENOMEM); if (!swap_reserve_by_cred(fork_charge, p->p_ucred)) { vmspace_free(newvmspace); return (ENOMEM); } PROC_VMSPACE_LOCK(p); p->p_vmspace = newvmspace; PROC_VMSPACE_UNLOCK(p); if (p == curthread->td_proc) pmap_activate(curthread); vmspace_free(oldvmspace); return (0); } /* * vm_map_lookup: * * Finds the VM object, offset, and * protection for a given virtual address in the * specified map, assuming a page fault of the * type specified. * * Leaves the map in question locked for read; return * values are guaranteed until a vm_map_lookup_done * call is performed. Note that the map argument * is in/out; the returned map must be used in * the call to vm_map_lookup_done. * * A handle (out_entry) is returned for use in * vm_map_lookup_done, to make that fast. * * If a lookup is requested with "write protection" * specified, the map may be changed to perform virtual * copying operations, although the data referenced will * remain the same. */ int vm_map_lookup(vm_map_t *var_map, /* IN/OUT */ vm_offset_t vaddr, vm_prot_t fault_typea, vm_map_entry_t *out_entry, /* OUT */ vm_object_t *object, /* OUT */ vm_pindex_t *pindex, /* OUT */ vm_prot_t *out_prot, /* OUT */ boolean_t *wired) /* OUT */ { vm_map_entry_t entry; vm_map_t map = *var_map; vm_prot_t prot; vm_prot_t fault_type = fault_typea; vm_object_t eobject; vm_size_t size; struct ucred *cred; RetryLookup: vm_map_lock_read(map); RetryLookupLocked: /* * Lookup the faulting address. */ if (!vm_map_lookup_entry(map, vaddr, out_entry)) { vm_map_unlock_read(map); return (KERN_INVALID_ADDRESS); } entry = *out_entry; /* * Handle submaps. */ if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) { vm_map_t old_map = map; *var_map = map = entry->object.sub_map; vm_map_unlock_read(old_map); goto RetryLookup; } /* * Check whether this task is allowed to have this page. */ prot = entry->protection; if ((fault_typea & VM_PROT_FAULT_LOOKUP) != 0) { fault_typea &= ~VM_PROT_FAULT_LOOKUP; if (prot == VM_PROT_NONE && map != kernel_map && (entry->eflags & MAP_ENTRY_GUARD) != 0 && (entry->eflags & (MAP_ENTRY_STACK_GAP_DN | MAP_ENTRY_STACK_GAP_UP)) != 0 && vm_map_growstack(map, vaddr, entry) == KERN_SUCCESS) goto RetryLookupLocked; } fault_type &= VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE; if ((fault_type & prot) != fault_type || prot == VM_PROT_NONE) { vm_map_unlock_read(map); return (KERN_PROTECTION_FAILURE); } KASSERT((prot & VM_PROT_WRITE) == 0 || (entry->eflags & (MAP_ENTRY_USER_WIRED | MAP_ENTRY_NEEDS_COPY)) != (MAP_ENTRY_USER_WIRED | MAP_ENTRY_NEEDS_COPY), ("entry %p flags %x", entry, entry->eflags)); if ((fault_typea & VM_PROT_COPY) != 0 && (entry->max_protection & VM_PROT_WRITE) == 0 && (entry->eflags & MAP_ENTRY_COW) == 0) { vm_map_unlock_read(map); return (KERN_PROTECTION_FAILURE); } /* * If this page is not pageable, we have to get it for all possible * accesses. */ *wired = (entry->wired_count != 0); if (*wired) fault_type = entry->protection; size = entry->end - entry->start; /* * If the entry was copy-on-write, we either ... */ if (entry->eflags & MAP_ENTRY_NEEDS_COPY) { /* * If we want to write the page, we may as well handle that * now since we've got the map locked. * * If we don't need to write the page, we just demote the * permissions allowed. */ if ((fault_type & VM_PROT_WRITE) != 0 || (fault_typea & VM_PROT_COPY) != 0) { /* * Make a new object, and place it in the object * chain. Note that no new references have appeared * -- one just moved from the map to the new * object. */ if (vm_map_lock_upgrade(map)) goto RetryLookup; if (entry->cred == NULL) { /* * The debugger owner is charged for * the memory. */ cred = curthread->td_ucred; crhold(cred); if (!swap_reserve_by_cred(size, cred)) { crfree(cred); vm_map_unlock(map); return (KERN_RESOURCE_SHORTAGE); } entry->cred = cred; } vm_object_shadow(&entry->object.vm_object, &entry->offset, size); entry->eflags &= ~MAP_ENTRY_NEEDS_COPY; eobject = entry->object.vm_object; if (eobject->cred != NULL) { /* * The object was not shadowed. */ swap_release_by_cred(size, entry->cred); crfree(entry->cred); entry->cred = NULL; } else if (entry->cred != NULL) { VM_OBJECT_WLOCK(eobject); eobject->cred = entry->cred; eobject->charge = size; VM_OBJECT_WUNLOCK(eobject); entry->cred = NULL; } vm_map_lock_downgrade(map); } else { /* * We're attempting to read a copy-on-write page -- * don't allow writes. */ prot &= ~VM_PROT_WRITE; } } /* * Create an object if necessary. */ if (entry->object.vm_object == NULL && !map->system_map) { if (vm_map_lock_upgrade(map)) goto RetryLookup; entry->object.vm_object = vm_object_allocate(OBJT_DEFAULT, atop(size)); entry->offset = 0; if (entry->cred != NULL) { VM_OBJECT_WLOCK(entry->object.vm_object); entry->object.vm_object->cred = entry->cred; entry->object.vm_object->charge = size; VM_OBJECT_WUNLOCK(entry->object.vm_object); entry->cred = NULL; } vm_map_lock_downgrade(map); } /* * Return the object/offset from this entry. If the entry was * copy-on-write or empty, it has been fixed up. */ *pindex = OFF_TO_IDX((vaddr - entry->start) + entry->offset); *object = entry->object.vm_object; *out_prot = prot; return (KERN_SUCCESS); } /* * vm_map_lookup_locked: * * Lookup the faulting address. A version of vm_map_lookup that returns * KERN_FAILURE instead of blocking on map lock or memory allocation. */ int vm_map_lookup_locked(vm_map_t *var_map, /* IN/OUT */ vm_offset_t vaddr, vm_prot_t fault_typea, vm_map_entry_t *out_entry, /* OUT */ vm_object_t *object, /* OUT */ vm_pindex_t *pindex, /* OUT */ vm_prot_t *out_prot, /* OUT */ boolean_t *wired) /* OUT */ { vm_map_entry_t entry; vm_map_t map = *var_map; vm_prot_t prot; vm_prot_t fault_type = fault_typea; /* * Lookup the faulting address. */ if (!vm_map_lookup_entry(map, vaddr, out_entry)) return (KERN_INVALID_ADDRESS); entry = *out_entry; /* * Fail if the entry refers to a submap. */ if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) return (KERN_FAILURE); /* * Check whether this task is allowed to have this page. */ prot = entry->protection; fault_type &= VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE; if ((fault_type & prot) != fault_type) return (KERN_PROTECTION_FAILURE); /* * If this page is not pageable, we have to get it for all possible * accesses. */ *wired = (entry->wired_count != 0); if (*wired) fault_type = entry->protection; if (entry->eflags & MAP_ENTRY_NEEDS_COPY) { /* * Fail if the entry was copy-on-write for a write fault. */ if (fault_type & VM_PROT_WRITE) return (KERN_FAILURE); /* * We're attempting to read a copy-on-write page -- * don't allow writes. */ prot &= ~VM_PROT_WRITE; } /* * Fail if an object should be created. */ if (entry->object.vm_object == NULL && !map->system_map) return (KERN_FAILURE); /* * Return the object/offset from this entry. If the entry was * copy-on-write or empty, it has been fixed up. */ *pindex = OFF_TO_IDX((vaddr - entry->start) + entry->offset); *object = entry->object.vm_object; *out_prot = prot; return (KERN_SUCCESS); } /* * vm_map_lookup_done: * * Releases locks acquired by a vm_map_lookup * (according to the handle returned by that lookup). */ void vm_map_lookup_done(vm_map_t map, vm_map_entry_t entry) { /* * Unlock the main-level map */ vm_map_unlock_read(map); } vm_offset_t vm_map_max_KBI(const struct vm_map *map) { return (vm_map_max(map)); } vm_offset_t vm_map_min_KBI(const struct vm_map *map) { return (vm_map_min(map)); } pmap_t vm_map_pmap_KBI(vm_map_t map) { return (map->pmap); } #ifdef INVARIANTS static void _vm_map_assert_consistent(vm_map_t map, int check) { vm_map_entry_t entry, prev; vm_size_t max_left, max_right; if (enable_vmmap_check != check) return; prev = &map->header; VM_MAP_ENTRY_FOREACH(entry, map) { KASSERT(prev->end <= entry->start, ("map %p prev->end = %jx, start = %jx", map, (uintmax_t)prev->end, (uintmax_t)entry->start)); KASSERT(entry->start < entry->end, ("map %p start = %jx, end = %jx", map, (uintmax_t)entry->start, (uintmax_t)entry->end)); KASSERT(entry->end <= vm_map_entry_succ(entry)->start, ("map %p end = %jx, next->start = %jx", map, (uintmax_t)entry->end, (uintmax_t)vm_map_entry_succ(entry)->start)); KASSERT(entry->left == NULL || entry->left->start < entry->start, ("map %p left->start = %jx, start = %jx", map, (uintmax_t)entry->left->start, (uintmax_t)entry->start)); KASSERT(entry->right == NULL || entry->start < entry->right->start, ("map %p start = %jx, right->start = %jx", map, (uintmax_t)entry->start, (uintmax_t)entry->right->start)); max_left = vm_map_entry_max_free_left(entry, vm_map_entry_pred(entry)); max_right = vm_map_entry_max_free_right(entry, vm_map_entry_succ(entry)); KASSERT(entry->max_free == MAX(max_left, max_right), ("map %p max = %jx, max_left = %jx, max_right = %jx", map, (uintmax_t)entry->max_free, (uintmax_t)max_left, (uintmax_t)max_right)); prev = entry; } KASSERT(prev->end <= entry->start, ("map %p prev->end = %jx, start = %jx", map, (uintmax_t)prev->end, (uintmax_t)entry->start)); } #endif #include "opt_ddb.h" #ifdef DDB #include #include static void vm_map_print(vm_map_t map) { vm_map_entry_t entry, prev; db_iprintf("Task map %p: pmap=%p, nentries=%d, version=%u\n", (void *)map, (void *)map->pmap, map->nentries, map->timestamp); db_indent += 2; prev = &map->header; VM_MAP_ENTRY_FOREACH(entry, map) { db_iprintf("map entry %p: start=%p, end=%p, eflags=%#x, \n", (void *)entry, (void *)entry->start, (void *)entry->end, entry->eflags); { static char *inheritance_name[4] = {"share", "copy", "none", "donate_copy"}; db_iprintf(" prot=%x/%x/%s", entry->protection, entry->max_protection, inheritance_name[(int)(unsigned char) entry->inheritance]); if (entry->wired_count != 0) db_printf(", wired"); } if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) { db_printf(", share=%p, offset=0x%jx\n", (void *)entry->object.sub_map, (uintmax_t)entry->offset); if (prev == &map->header || prev->object.sub_map != entry->object.sub_map) { db_indent += 2; vm_map_print((vm_map_t)entry->object.sub_map); db_indent -= 2; } } else { if (entry->cred != NULL) db_printf(", ruid %d", entry->cred->cr_ruid); db_printf(", object=%p, offset=0x%jx", (void *)entry->object.vm_object, (uintmax_t)entry->offset); if (entry->object.vm_object && entry->object.vm_object->cred) db_printf(", obj ruid %d charge %jx", entry->object.vm_object->cred->cr_ruid, (uintmax_t)entry->object.vm_object->charge); if (entry->eflags & MAP_ENTRY_COW) db_printf(", copy (%s)", (entry->eflags & MAP_ENTRY_NEEDS_COPY) ? "needed" : "done"); db_printf("\n"); if (prev == &map->header || prev->object.vm_object != entry->object.vm_object) { db_indent += 2; vm_object_print((db_expr_t)(intptr_t) entry->object.vm_object, 0, 0, (char *)0); db_indent -= 2; } } prev = entry; } db_indent -= 2; } DB_SHOW_COMMAND(map, map) { if (!have_addr) { db_printf("usage: show map \n"); return; } vm_map_print((vm_map_t)addr); } DB_SHOW_COMMAND(procvm, procvm) { struct proc *p; if (have_addr) { p = db_lookup_proc(addr); } else { p = curproc; } db_printf("p = %p, vmspace = %p, map = %p, pmap = %p\n", (void *)p, (void *)p->p_vmspace, (void *)&p->p_vmspace->vm_map, (void *)vmspace_pmap(p->p_vmspace)); vm_map_print((vm_map_t)&p->p_vmspace->vm_map); } #endif /* DDB */