diff --git a/lib/libc/sys/mmap.2 b/lib/libc/sys/mmap.2 index 5a3c6eb5d06e..37006b0355a6 100644 --- a/lib/libc/sys/mmap.2 +++ b/lib/libc/sys/mmap.2 @@ -1,568 +1,567 @@ .\" Copyright (c) 1991, 1993 .\" The Regents of the University of California. All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" 3. Neither the name of the University nor the names of its contributors .\" may be used to endorse or promote products derived from this software .\" without specific prior written permission. .\" .\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE .\" ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE .\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS .\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) .\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT .\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" .\" @(#)mmap.2 8.4 (Berkeley) 5/11/95 .\" $FreeBSD$ .\" -.Dd December 24, 2020 +.Dd August 14, 2023 .Dt MMAP 2 .Os .Sh NAME .Nm mmap .Nd allocate memory, or map files or devices into memory .Sh LIBRARY .Lb libc .Sh SYNOPSIS .In sys/mman.h .Ft void * .Fn mmap "void *addr" "size_t len" "int prot" "int flags" "int fd" "off_t offset" .Sh DESCRIPTION The .Fn mmap system call causes the pages starting at .Fa addr and continuing for at most .Fa len bytes to be mapped from the object described by .Fa fd , starting at byte offset .Fa offset . If .Fa len is not a multiple of the page size, the mapped region may extend past the specified range. Any such extension beyond the end of the mapped object will be zero-filled. .Pp If .Fa fd references a regular file or a shared memory object, the range of bytes starting at .Fa offset and continuing for .Fa len bytes must be legitimate for the possible (not necessarily current) offsets in the object. In particular, the .Fa offset value cannot be negative. If the object is truncated and the process later accesses a page that is wholly within the truncated region, the access is aborted and a .Dv SIGBUS signal is delivered to the process. .Pp If .Fa fd references a device file, the interpretation of the .Fa offset value is device specific and defined by the device driver. The virtual memory subsystem does not impose any restrictions on the .Fa offset value in this case, passing it unchanged to the driver. .Pp If .Fa addr is non-zero, it is used as a hint to the system. (As a convenience to the system, the actual address of the region may differ from the address supplied.) If .Fa addr is zero, an address will be selected by the system. The actual starting address of the region is returned. A successful .Fa mmap deletes any previous mapping in the allocated address range. .Pp The protections (region accessibility) are specified in the .Fa prot argument by .Em or Ns 'ing the following values: .Pp .Bl -tag -width PROT_WRITE -compact .It Dv PROT_NONE Pages may not be accessed. .It Dv PROT_READ Pages may be read. .It Dv PROT_WRITE Pages may be written. .It Dv PROT_EXEC Pages may be executed. .El .Pp In addition to these protection flags, .Fx provides the ability to set the maximum protection of a region allocated by .Nm and later altered by .Xr mprotect 2 . This is accomplished by .Em or Ns 'ing one or more .Dv PROT_ values wrapped in the .Dv PROT_MAX() macro into the .Fa prot argument. .Pp The .Fa flags argument specifies the type of the mapped object, mapping options and whether modifications made to the mapped copy of the page are private to the process or are to be shared with other references. Sharing, mapping type and options are specified in the .Fa flags argument by .Em or Ns 'ing the following values: .Bl -tag -width MAP_PREFAULT_READ .It Dv MAP_32BIT Request a region in the first 2GB of the current process's address space. If a suitable region cannot be found, .Fn mmap will fail. -This flag is only available on 64-bit platforms. .It Dv MAP_ALIGNED Ns Pq Fa n Align the region on a requested boundary. If a suitable region cannot be found, .Fn mmap will fail. The .Fa n argument specifies the binary logarithm of the desired alignment. .It Dv MAP_ALIGNED_SUPER Align the region to maximize the potential use of large .Pq Dq super pages. If a suitable region cannot be found, .Fn mmap will fail. The system will choose a suitable page size based on the size of mapping. The page size used as well as the alignment of the region may both be affected by properties of the file being mapped. In particular, the physical address of existing pages of a file may require a specific alignment. The region is not guaranteed to be aligned on any specific boundary. .It Dv MAP_ANON Map anonymous memory not associated with any specific file. The file descriptor used for creating .Dv MAP_ANON must be \-1. The .Fa offset argument must be 0. .\".It Dv MAP_FILE .\"Mapped from a regular file or character-special device memory. .It Dv MAP_ANONYMOUS This flag is identical to .Dv MAP_ANON and is provided for compatibility. .It Dv MAP_EXCL This flag can only be used in combination with .Dv MAP_FIXED . Please see the definition of .Dv MAP_FIXED for the description of its effect. .It Dv MAP_FIXED Do not permit the system to select a different address than the one specified. If the specified address cannot be used, .Fn mmap will fail. If .Dv MAP_FIXED is specified, .Fa addr must be a multiple of the page size. If .Dv MAP_EXCL is not specified, a successful .Dv MAP_FIXED request replaces any previous mappings for the process' pages in the range from .Fa addr to .Fa addr + .Fa len . In contrast, if .Dv MAP_EXCL is specified, the request will fail if a mapping already exists within the range. .It Dv MAP_GUARD Instead of a mapping, create a guard of the specified size. Guards allow a process to create reservations in its address space, which can later be replaced by actual mappings. .Pp .Fa mmap will not create mappings in the address range of a guard unless the request specifies .Dv MAP_FIXED . Guards can be destroyed with .Xr munmap 2 . Any memory access by a thread to the guarded range results in the delivery of a .Dv SIGSEGV signal to that thread. .It Dv MAP_NOCORE Region is not included in a core file. .It Dv MAP_NOSYNC Causes data dirtied via this VM map to be flushed to physical media only when necessary (usually by the pager) rather than gratuitously. Typically this prevents the update daemons from flushing pages dirtied through such maps and thus allows efficient sharing of memory across unassociated processes using a file-backed shared memory map. Without this option any VM pages you dirty may be flushed to disk every so often (every 30-60 seconds usually) which can create performance problems if you do not need that to occur (such as when you are using shared file-backed mmap regions for IPC purposes). Dirty data will be flushed automatically when all mappings of an object are removed and all descriptors referencing the object are closed. Note that VM/file system coherency is maintained whether you use .Dv MAP_NOSYNC or not. This option is not portable across .Ux platforms (yet), though some may implement the same behavior by default. .Pp .Em WARNING ! Extending a file with .Xr ftruncate 2 , thus creating a big hole, and then filling the hole by modifying a shared .Fn mmap can lead to severe file fragmentation. In order to avoid such fragmentation you should always pre-allocate the file's backing store by .Fn write Ns ing zero's into the newly extended area prior to modifying the area via your .Fn mmap . The fragmentation problem is especially sensitive to .Dv MAP_NOSYNC pages, because pages may be flushed to disk in a totally random order. .Pp The same applies when using .Dv MAP_NOSYNC to implement a file-based shared memory store. It is recommended that you create the backing store by .Fn write Ns ing zero's to the backing file rather than .Fn ftruncate Ns ing it. You can test file fragmentation by observing the KB/t (kilobytes per transfer) results from an .Dq Li iostat 1 while reading a large file sequentially, e.g.,\& using .Dq Li dd if=filename of=/dev/null bs=32k . .Pp The .Xr fsync 2 system call will flush all dirty data and metadata associated with a file, including dirty NOSYNC VM data, to physical media. The .Xr sync 8 command and .Xr sync 2 system call generally do not flush dirty NOSYNC VM data. The .Xr msync 2 system call is usually not needed since .Bx implements a coherent file system buffer cache. However, it may be used to associate dirty VM pages with file system buffers and thus cause them to be flushed to physical media sooner rather than later. .It Dv MAP_PREFAULT_READ Immediately update the calling process's lowest-level virtual address translation structures, such as its page table, so that every memory resident page within the region is mapped for read access. Ordinarily these structures are updated lazily. The effect of this option is to eliminate any soft faults that would otherwise occur on the initial read accesses to the region. Although this option does not preclude .Fa prot from including .Dv PROT_WRITE , it does not eliminate soft faults on the initial write accesses to the region. .It Dv MAP_PRIVATE Modifications are private. .It Dv MAP_SHARED Modifications are shared. .It Dv MAP_STACK Creates both a mapped region that grows downward on demand and an adjoining guard that both reserves address space for the mapped region to grow into and limits the mapped region's growth. Together, the mapped region and the guard occupy .Fa len bytes of the address space. The guard starts at the returned address, and the mapped region ends at the returned address plus .Fa len bytes. Upon access to the guard, the mapped region automatically grows in size, and the guard shrinks by an equal amount. Essentially, the boundary between the guard and the mapped region moves downward so that the access falls within the enlarged mapped region. However, the guard will never shrink to less than the number of pages specified by the sysctl .Dv security.bsd.stack_guard_page , thereby ensuring that a gap for detecting stack overflow always exists between the downward growing mapped region and the closest mapped region beneath it. .Pp .Dv MAP_STACK implies .Dv MAP_ANON and .Fa offset of 0. The .Fa fd argument must be -1 and .Fa prot must include at least .Dv PROT_READ and .Dv PROT_WRITE . The size of the guard, in pages, is specified by sysctl .Dv security.bsd.stack_guard_page . .El .Pp The .Xr close 2 system call does not unmap pages, see .Xr munmap 2 for further information. .Sh NOTES Although this implementation does not impose any alignment restrictions on the .Fa offset argument, a portable program must only use page-aligned values. .Pp Large page mappings require that the pages backing an object be aligned in matching blocks in both the virtual address space and RAM. The system will automatically attempt to use large page mappings when mapping an object that is already backed by large pages in RAM by aligning the mapping request in the virtual address space to match the alignment of the large physical pages. The system may also use large page mappings when mapping portions of an object that are not yet backed by pages in RAM. The .Dv MAP_ALIGNED_SUPER flag is an optimization that will align the mapping request to the size of a large page similar to .Dv MAP_ALIGNED , except that the system will override this alignment if an object already uses large pages so that the mapping will be consistent with the existing large pages. This flag is mostly useful for maximizing the use of large pages on the first mapping of objects that do not yet have pages present in RAM. .Sh RETURN VALUES Upon successful completion, .Fn mmap returns a pointer to the mapped region. Otherwise, a value of .Dv MAP_FAILED is returned and .Va errno is set to indicate the error. .Sh ERRORS The .Fn mmap system call will fail if: .Bl -tag -width Er .It Bq Er EACCES The flag .Dv PROT_READ was specified as part of the .Fa prot argument and .Fa fd was not open for reading. The flags .Dv MAP_SHARED and .Dv PROT_WRITE were specified as part of the .Fa flags and .Fa prot argument and .Fa fd was not open for writing. .It Bq Er EBADF The .Fa fd argument is not a valid open file descriptor. .It Bq Er EINVAL An invalid (negative) value was passed in the .Fa offset argument, when .Fa fd referenced a regular file or shared memory. .It Bq Er EINVAL An invalid value was passed in the .Fa prot argument. .It Bq Er EINVAL An undefined option was set in the .Fa flags argument. .It Bq Er EINVAL Both .Dv MAP_PRIVATE and .Dv MAP_SHARED were specified. .It Bq Er EINVAL None of .Dv MAP_ANON , .Dv MAP_GUARD , .Dv MAP_PRIVATE , .Dv MAP_SHARED , or .Dv MAP_STACK was specified. At least one of these flags must be included. .It Bq Er EINVAL .Dv MAP_STACK was specified and .Va len is less than or equal to the guard size. .It Bq Er EINVAL .Dv MAP_FIXED was specified and the .Fa addr argument was not page aligned, or part of the desired address space resides out of the valid address space for a user process. .It Bq Er EINVAL Both .Dv MAP_FIXED and .Dv MAP_32BIT were specified and part of the desired address space resides outside of the first 2GB of user address space. .It Bq Er EINVAL The .Fa len argument was equal to zero. .It Bq Er EINVAL .Dv MAP_ALIGNED was specified and the desired alignment was either larger than the virtual address size of the machine or smaller than a page. .It Bq Er EINVAL .Dv MAP_ANON was specified and the .Fa fd argument was not -1. .It Bq Er EINVAL .Dv MAP_ANON was specified and the .Fa offset argument was not 0. .It Bq Er EINVAL Both .Dv MAP_FIXED and .Dv MAP_EXCL were specified, but the requested region is already used by a mapping. .It Bq Er EINVAL .Dv MAP_EXCL was specified, but .Dv MAP_FIXED was not. .It Bq Er EINVAL .Dv MAP_GUARD was specified, but the .Fa offset argument was not zero, the .Fa fd argument was not -1, or the .Fa prot argument was not .Dv PROT_NONE . .It Bq Er EINVAL .Dv MAP_GUARD was specified together with one of the flags .Dv MAP_ANON , .Dv MAP_PREFAULT , .Dv MAP_PREFAULT_READ , .Dv MAP_PRIVATE , .Dv MAP_SHARED , .Dv MAP_STACK . .It Bq Er ENODEV .Dv MAP_ANON has not been specified and .Fa fd did not reference a regular or character special file. .It Bq Er ENOMEM .Dv MAP_FIXED was specified and the .Fa addr argument was not available. .Dv MAP_ANON was specified and insufficient memory was available. .It Bq Er ENOTSUP The .Fa prot argument contains protections which are not a subset of the specified maximum protections. .El .Sh SEE ALSO .Xr madvise 2 , .Xr mincore 2 , .Xr minherit 2 , .Xr mlock 2 , .Xr mprotect 2 , .Xr msync 2 , .Xr munlock 2 , .Xr munmap 2 , .Xr getpagesize 3 , .Xr getpagesizes 3 .Sh HISTORY The .Nm system call was first documented in .Bx 4.2 and implemented in .Bx 4.4 . .\" XXX: lots of missing history of FreeBSD additions. .Pp The .Dv PROT_MAX functionality was introduced in .Fx 13 . diff --git a/lib/libsysdecode/flags.c b/lib/libsysdecode/flags.c index ac109cc454e0..31c125ad71af 100644 --- a/lib/libsysdecode/flags.c +++ b/lib/libsysdecode/flags.c @@ -1,1204 +1,1195 @@ /* * Copyright (c) 2006 "David Kirchner" . All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #define L2CAP_SOCKET_CHECKED #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "support.h" #define X(a) { a, #a }, #define XEND { 0, NULL } #define TABLE_START(n) static struct name_table n[] = { #define TABLE_ENTRY X #define TABLE_END XEND }; #include "tables.h" #undef TABLE_START #undef TABLE_ENTRY #undef TABLE_END const char * sysdecode_atfd(int fd) { if (fd == AT_FDCWD) return ("AT_FDCWD"); return (NULL); } bool sysdecode_atflags(FILE *fp, int flag, int *rem) { return (print_mask_int(fp, atflags, flag, rem)); } static struct name_table semctlops[] = { X(GETNCNT) X(GETPID) X(GETVAL) X(GETALL) X(GETZCNT) X(SETVAL) X(SETALL) X(IPC_RMID) X(IPC_SET) X(IPC_STAT) XEND }; const char * sysdecode_semctl_cmd(int cmd) { return (lookup_value(semctlops, cmd)); } static struct name_table shmctlops[] = { X(IPC_RMID) X(IPC_SET) X(IPC_STAT) XEND }; const char * sysdecode_shmctl_cmd(int cmd) { return (lookup_value(shmctlops, cmd)); } const char * sysdecode_msgctl_cmd(int cmd) { return (sysdecode_shmctl_cmd(cmd)); } static struct name_table semgetflags[] = { X(IPC_CREAT) X(IPC_EXCL) X(SEM_R) X(SEM_A) X((SEM_R>>3)) X((SEM_A>>3)) X((SEM_R>>6)) X((SEM_A>>6)) XEND }; bool sysdecode_semget_flags(FILE *fp, int flag, int *rem) { return (print_mask_int(fp, semgetflags, flag, rem)); } static struct name_table idtypes[] = { X(P_PID) X(P_PPID) X(P_PGID) X(P_SID) X(P_CID) X(P_UID) X(P_GID) X(P_ALL) X(P_LWPID) X(P_TASKID) X(P_PROJID) X(P_POOLID) X(P_JAILID) X(P_CTID) X(P_CPUID) X(P_PSETID) XEND }; /* XXX: idtype is really an idtype_t */ const char * sysdecode_idtype(int idtype) { return (lookup_value(idtypes, idtype)); } /* * [g|s]etsockopt's level argument can either be SOL_SOCKET or a * protocol-specific value. */ const char * sysdecode_sockopt_level(int level) { const char *str; if (level == SOL_SOCKET) return ("SOL_SOCKET"); /* SOL_* constants for Bluetooth sockets. */ str = lookup_value(ngbtsolevel, level); if (str != NULL) return (str); /* * IP and Infiniband sockets use IP protocols as levels. Not all * protocols are valid but it is simpler to just allow all of them. * * XXX: IPPROTO_IP == 0, but UNIX domain sockets use a level of 0 * for private options. */ str = sysdecode_ipproto(level); if (str != NULL) return (str); return (NULL); } bool sysdecode_vmprot(FILE *fp, int type, int *rem) { return (print_mask_int(fp, vmprot, type, rem)); } static struct name_table sockflags[] = { X(SOCK_CLOEXEC) X(SOCK_NONBLOCK) XEND }; bool sysdecode_socket_type(FILE *fp, int type, int *rem) { const char *str; uintmax_t val; bool printed; str = lookup_value(socktype, type & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)); if (str != NULL) { fputs(str, fp); *rem = 0; printed = true; } else { *rem = type & ~(SOCK_CLOEXEC | SOCK_NONBLOCK); printed = false; } val = type & (SOCK_CLOEXEC | SOCK_NONBLOCK); print_mask_part(fp, sockflags, &val, &printed); return (printed); } bool sysdecode_access_mode(FILE *fp, int mode, int *rem) { return (print_mask_int(fp, accessmode, mode, rem)); } /* XXX: 'type' is really an acl_type_t. */ const char * sysdecode_acltype(int type) { return (lookup_value(acltype, type)); } bool sysdecode_cap_fcntlrights(FILE *fp, uint32_t rights, uint32_t *rem) { return (print_mask_int(fp, capfcntl, rights, rem)); } bool sysdecode_close_range_flags(FILE *fp, int flags, int *rem) { return (print_mask_int(fp, closerangeflags, flags, rem)); } const char * sysdecode_extattrnamespace(int namespace) { return (lookup_value(extattrns, namespace)); } const char * sysdecode_fadvice(int advice) { return (lookup_value(fadvisebehav, advice)); } bool sysdecode_open_flags(FILE *fp, int flags, int *rem) { bool printed; int mode; uintmax_t val; mode = flags & O_ACCMODE; flags &= ~O_ACCMODE; switch (mode) { case O_RDONLY: if (flags & O_EXEC) { flags &= ~O_EXEC; fputs("O_EXEC", fp); } else fputs("O_RDONLY", fp); printed = true; mode = 0; break; case O_WRONLY: fputs("O_WRONLY", fp); printed = true; mode = 0; break; case O_RDWR: fputs("O_RDWR", fp); printed = true; mode = 0; break; default: printed = false; } val = (unsigned)flags; print_mask_part(fp, openflags, &val, &printed); if (rem != NULL) *rem = val | mode; return (printed); } bool sysdecode_fcntl_fileflags(FILE *fp, int flags, int *rem) { bool printed; int oflags; /* * The file flags used with F_GETFL/F_SETFL mostly match the * flags passed to open(2). However, a few open-only flag * bits have been repurposed for fcntl-only flags. */ oflags = flags & ~(O_NOFOLLOW | FRDAHEAD); printed = sysdecode_open_flags(fp, oflags, rem); if (flags & O_NOFOLLOW) { fprintf(fp, "%sFPOIXSHM", printed ? "|" : ""); printed = true; } if (flags & FRDAHEAD) { fprintf(fp, "%sFRDAHEAD", printed ? "|" : ""); printed = true; } return (printed); } bool sysdecode_flock_operation(FILE *fp, int operation, int *rem) { return (print_mask_int(fp, flockops, operation, rem)); } static struct name_table getfsstatmode[] = { X(MNT_WAIT) X(MNT_NOWAIT) XEND }; const char * sysdecode_getfsstat_mode(int mode) { return (lookup_value(getfsstatmode, mode)); } const char * sysdecode_getrusage_who(int who) { return (lookup_value(rusage, who)); } static struct name_table kevent_user_ffctrl[] = { X(NOTE_FFNOP) X(NOTE_FFAND) X(NOTE_FFOR) X(NOTE_FFCOPY) XEND }; static struct name_table kevent_rdwr_fflags[] = { X(NOTE_LOWAT) X(NOTE_FILE_POLL) XEND }; static struct name_table kevent_vnode_fflags[] = { X(NOTE_DELETE) X(NOTE_WRITE) X(NOTE_EXTEND) X(NOTE_ATTRIB) X(NOTE_LINK) X(NOTE_RENAME) X(NOTE_REVOKE) X(NOTE_OPEN) X(NOTE_CLOSE) X(NOTE_CLOSE_WRITE) X(NOTE_READ) XEND }; static struct name_table kevent_proc_fflags[] = { X(NOTE_EXIT) X(NOTE_FORK) X(NOTE_EXEC) X(NOTE_TRACK) X(NOTE_TRACKERR) X(NOTE_CHILD) XEND }; static struct name_table kevent_timer_fflags[] = { X(NOTE_SECONDS) X(NOTE_MSECONDS) X(NOTE_USECONDS) X(NOTE_NSECONDS) X(NOTE_ABSTIME) XEND }; void sysdecode_kevent_fflags(FILE *fp, short filter, int fflags, int base) { int rem; if (fflags == 0) { fputs("0", fp); return; } switch (filter) { case EVFILT_READ: case EVFILT_WRITE: if (!print_mask_int(fp, kevent_rdwr_fflags, fflags, &rem)) fprintf(fp, "%#x", rem); else if (rem != 0) fprintf(fp, "|%#x", rem); break; case EVFILT_VNODE: if (!print_mask_int(fp, kevent_vnode_fflags, fflags, &rem)) fprintf(fp, "%#x", rem); else if (rem != 0) fprintf(fp, "|%#x", rem); break; case EVFILT_PROC: case EVFILT_PROCDESC: if (!print_mask_int(fp, kevent_proc_fflags, fflags, &rem)) fprintf(fp, "%#x", rem); else if (rem != 0) fprintf(fp, "|%#x", rem); break; case EVFILT_TIMER: if (!print_mask_int(fp, kevent_timer_fflags, fflags, &rem)) fprintf(fp, "%#x", rem); else if (rem != 0) fprintf(fp, "|%#x", rem); break; case EVFILT_USER: { unsigned int ctrl, data; ctrl = fflags & NOTE_FFCTRLMASK; data = fflags & NOTE_FFLAGSMASK; if (fflags & NOTE_TRIGGER) { fputs("NOTE_TRIGGER", fp); if (fflags == NOTE_TRIGGER) return; fputc('|', fp); } /* * An event with 'ctrl' == NOTE_FFNOP is either a reported * (output) event for which only 'data' should be output * or a pointless input event. Assume that pointless * input events don't occur in practice. An event with * NOTE_TRIGGER is always an input event. */ if (ctrl != NOTE_FFNOP || fflags & NOTE_TRIGGER) { fprintf(fp, "%s|%#x", lookup_value(kevent_user_ffctrl, ctrl), data); } else { print_integer(fp, data, base); } break; } default: print_integer(fp, fflags, base); break; } } bool sysdecode_kevent_flags(FILE *fp, int flags, int *rem) { return (print_mask_int(fp, keventflags, flags, rem)); } const char * sysdecode_kevent_filter(int filter) { return (lookup_value(keventfilters, filter)); } const char * sysdecode_kldsym_cmd(int cmd) { return (lookup_value(kldsymcmd, cmd)); } const char * sysdecode_kldunload_flags(int flags) { return (lookup_value(kldunloadfflags, flags)); } const char * sysdecode_lio_listio_mode(int mode) { return (lookup_value(lio_listiomodes, mode)); } const char * sysdecode_madvice(int advice) { return (lookup_value(madvisebehav, advice)); } const char * sysdecode_minherit_inherit(int inherit) { return (lookup_value(minheritflags, inherit)); } bool sysdecode_mlockall_flags(FILE *fp, int flags, int *rem) { return (print_mask_int(fp, mlockallflags, flags, rem)); } bool sysdecode_mmap_prot(FILE *fp, int prot, int *rem) { int protm; bool printed; printed = false; protm = PROT_MAX_EXTRACT(prot); prot &= ~PROT_MAX(protm); if (protm != 0) { fputs("PROT_MAX(", fp); printed = print_mask_int(fp, mmapprot, protm, rem); fputs(")|", fp); } return (print_mask_int(fp, mmapprot, prot, rem) || printed); } bool sysdecode_fileflags(FILE *fp, fflags_t flags, fflags_t *rem) { return (print_mask_0(fp, fileflags, flags, rem)); } bool sysdecode_filemode(FILE *fp, int mode, int *rem) { return (print_mask_0(fp, filemode, mode, rem)); } bool sysdecode_mount_flags(FILE *fp, int flags, int *rem) { return (print_mask_int(fp, mountflags, flags, rem)); } bool sysdecode_msync_flags(FILE *fp, int flags, int *rem) { return (print_mask_int(fp, msyncflags, flags, rem)); } const char * sysdecode_nfssvc_flags(int flags) { return (lookup_value(nfssvcflags, flags)); } static struct name_table pipe2flags[] = { X(O_CLOEXEC) X(O_NONBLOCK) XEND }; bool sysdecode_pipe2_flags(FILE *fp, int flags, int *rem) { return (print_mask_0(fp, pipe2flags, flags, rem)); } const char * sysdecode_prio_which(int which) { return (lookup_value(prio, which)); } const char * sysdecode_procctl_cmd(int cmd) { return (lookup_value(procctlcmd, cmd)); } const char * sysdecode_ptrace_request(int request) { return (lookup_value(ptraceop, request)); } static struct name_table quotatypes[] = { X(GRPQUOTA) X(USRQUOTA) XEND }; bool sysdecode_quotactl_cmd(FILE *fp, int cmd) { const char *primary, *type; primary = lookup_value(quotactlcmds, cmd >> SUBCMDSHIFT); if (primary == NULL) return (false); fprintf(fp, "QCMD(%s,", primary); type = lookup_value(quotatypes, cmd & SUBCMDMASK); if (type != NULL) fprintf(fp, "%s", type); else fprintf(fp, "%#x", cmd & SUBCMDMASK); fprintf(fp, ")"); return (true); } bool sysdecode_reboot_howto(FILE *fp, int howto, int *rem) { bool printed; /* * RB_AUTOBOOT is special in that its value is zero, but it is * also an implied argument if a different operation is not * requested via RB_HALT, RB_POWERCYCLE, RB_POWEROFF, or * RB_REROOT. */ if (howto != 0 && (howto & (RB_HALT | RB_POWEROFF | RB_REROOT | RB_POWERCYCLE)) == 0) { fputs("RB_AUTOBOOT|", fp); printed = true; } else printed = false; return (print_mask_int(fp, rebootopt, howto, rem) || printed); } bool sysdecode_rfork_flags(FILE *fp, int flags, int *rem) { return (print_mask_int(fp, rforkflags, flags, rem)); } const char * sysdecode_rlimit(int resource) { return (lookup_value(rlimit, resource)); } const char * sysdecode_scheduler_policy(int policy) { return (lookup_value(schedpolicy, policy)); } bool sysdecode_sendfile_flags(FILE *fp, int flags, int *rem) { return (print_mask_int(fp, sendfileflags, flags, rem)); } bool sysdecode_shmat_flags(FILE *fp, int flags, int *rem) { return (print_mask_int(fp, shmatflags, flags, rem)); } const char * sysdecode_shutdown_how(int how) { return (lookup_value(shutdownhow, how)); } const char * sysdecode_sigbus_code(int si_code) { return (lookup_value(sigbuscode, si_code)); } const char * sysdecode_sigchld_code(int si_code) { return (lookup_value(sigchldcode, si_code)); } const char * sysdecode_sigfpe_code(int si_code) { return (lookup_value(sigfpecode, si_code)); } const char * sysdecode_sigill_code(int si_code) { return (lookup_value(sigillcode, si_code)); } const char * sysdecode_sigsegv_code(int si_code) { return (lookup_value(sigsegvcode, si_code)); } const char * sysdecode_sigtrap_code(int si_code) { return (lookup_value(sigtrapcode, si_code)); } const char * sysdecode_sigprocmask_how(int how) { return (lookup_value(sigprocmaskhow, how)); } const char * sysdecode_socketdomain(int domain) { return (lookup_value(sockdomain, domain)); } const char * sysdecode_socket_protocol(int domain, int protocol) { switch (domain) { case PF_INET: case PF_INET6: return (lookup_value(sockipproto, protocol)); default: return (NULL); } } const char * sysdecode_sockaddr_family(int sa_family) { return (lookup_value(sockfamily, sa_family)); } const char * sysdecode_ipproto(int protocol) { return (lookup_value(sockipproto, protocol)); } const char * sysdecode_sockopt_name(int level, int optname) { if (level == SOL_SOCKET) return (lookup_value(sockopt, optname)); if (level == IPPROTO_IP) /* XXX: UNIX domain socket options use a level of 0 also. */ return (lookup_value(sockoptip, optname)); if (level == IPPROTO_IPV6) return (lookup_value(sockoptipv6, optname)); if (level == IPPROTO_SCTP) return (lookup_value(sockoptsctp, optname)); if (level == IPPROTO_TCP) return (lookup_value(sockopttcp, optname)); if (level == IPPROTO_UDP) return (lookup_value(sockoptudp, optname)); if (level == IPPROTO_UDPLITE) return (lookup_value(sockoptudplite, optname)); return (NULL); } bool sysdecode_thr_create_flags(FILE *fp, int flags, int *rem) { return (print_mask_int(fp, thrcreateflags, flags, rem)); } const char * sysdecode_umtx_op(int op) { return (lookup_value(umtxop, op)); } bool sysdecode_umtx_op_flags(FILE *fp, int op, int *rem) { uintmax_t val; bool printed; printed = false; val = (unsigned)op; print_mask_part(fp, umtxopflags, &val, &printed); if (rem != NULL) *rem = val; return (printed); } const char * sysdecode_vmresult(int result) { return (lookup_value(vmresult, result)); } bool sysdecode_wait4_options(FILE *fp, int options, int *rem) { bool printed; int opt6; /* A flags value of 0 is normal. */ if (options == 0) { fputs("0", fp); if (rem != NULL) *rem = 0; return (true); } /* * These flags are implicit and aren't valid flags for wait4() * directly (though they don't fail with EINVAL). */ opt6 = options & (WEXITED | WTRAPPED); options &= ~opt6; printed = print_mask_int(fp, wait6opt, options, rem); if (rem != NULL) *rem |= opt6; return (printed); } bool sysdecode_wait6_options(FILE *fp, int options, int *rem) { return (print_mask_int(fp, wait6opt, options, rem)); } const char * sysdecode_whence(int whence) { return (lookup_value(seekwhence, whence)); } const char * sysdecode_fcntl_cmd(int cmd) { return (lookup_value(fcntlcmd, cmd)); } static struct name_table fcntl_fd_arg[] = { X(FD_CLOEXEC) X(0) XEND }; bool sysdecode_fcntl_arg_p(int cmd) { switch (cmd) { case F_GETFD: case F_GETFL: case F_GETOWN: return (false); default: return (true); } } void sysdecode_fcntl_arg(FILE *fp, int cmd, uintptr_t arg, int base) { int rem; switch (cmd) { case F_SETFD: if (!print_value(fp, fcntl_fd_arg, arg)) print_integer(fp, arg, base); break; case F_SETFL: if (!sysdecode_fcntl_fileflags(fp, arg, &rem)) fprintf(fp, "%#x", rem); else if (rem != 0) fprintf(fp, "|%#x", rem); break; case F_GETLK: case F_SETLK: case F_SETLKW: fprintf(fp, "%p", (void *)arg); break; default: print_integer(fp, arg, base); break; } } bool sysdecode_mmap_flags(FILE *fp, int flags, int *rem) { uintmax_t val; bool printed; int align; /* * MAP_ALIGNED can't be handled directly by print_mask_int(). - * MAP_32BIT is also problematic since it isn't defined for - * all platforms. */ printed = false; align = flags & MAP_ALIGNMENT_MASK; val = (unsigned)flags & ~MAP_ALIGNMENT_MASK; print_mask_part(fp, mmapflags, &val, &printed); -#ifdef MAP_32BIT - if (val & MAP_32BIT) { - fprintf(fp, "%sMAP_32BIT", printed ? "|" : ""); - printed = true; - val &= ~MAP_32BIT; - } -#endif if (align != 0) { if (printed) fputc('|', fp); if (align == MAP_ALIGNED_SUPER) fputs("MAP_ALIGNED_SUPER", fp); else fprintf(fp, "MAP_ALIGNED(%d)", align >> MAP_ALIGNMENT_SHIFT); printed = true; } if (rem != NULL) *rem = val; return (printed); } const char * sysdecode_pathconf_name(int name) { return (lookup_value(pathconfname, name)); } const char * sysdecode_rtprio_function(int function) { return (lookup_value(rtpriofuncs, function)); } bool sysdecode_msg_flags(FILE *fp, int flags, int *rem) { return (print_mask_0(fp, msgflags, flags, rem)); } const char * sysdecode_sigcode(int sig, int si_code) { const char *str; str = lookup_value(sigcode, si_code); if (str != NULL) return (str); switch (sig) { case SIGILL: return (sysdecode_sigill_code(si_code)); case SIGBUS: return (sysdecode_sigbus_code(si_code)); case SIGSEGV: return (sysdecode_sigsegv_code(si_code)); case SIGFPE: return (sysdecode_sigfpe_code(si_code)); case SIGTRAP: return (sysdecode_sigtrap_code(si_code)); case SIGCHLD: return (sysdecode_sigchld_code(si_code)); default: return (NULL); } } const char * sysdecode_sysarch_number(int number) { return (lookup_value(sysarchnum, number)); } bool sysdecode_umtx_cvwait_flags(FILE *fp, u_long flags, u_long *rem) { return (print_mask_0ul(fp, umtxcvwaitflags, flags, rem)); } bool sysdecode_umtx_rwlock_flags(FILE *fp, u_long flags, u_long *rem) { return (print_mask_0ul(fp, umtxrwlockflags, flags, rem)); } void sysdecode_cap_rights(FILE *fp, cap_rights_t *rightsp) { cap_rights_t diff, sum, zero; const struct name_table *t; int i; bool comma; for (i = 0; i < CAPARSIZE(rightsp); i++) { if (CAPIDXBIT(rightsp->cr_rights[i]) != 1 << i) { fprintf(fp, "invalid cap_rights_t"); return; } } cap_rights_init(&sum); diff = *rightsp; for (t = caprights, comma = false; t->str != NULL; t++) { if (cap_rights_is_set(rightsp, t->val)) { cap_rights_clear(&diff, t->val); if (cap_rights_is_set(&sum, t->val)) { /* Don't print redundant rights. */ continue; } cap_rights_set(&sum, t->val); fprintf(fp, "%s%s", comma ? "," : "", t->str); comma = true; } } if (!comma) fprintf(fp, "CAP_NONE"); /* * Provide a breadcrumb if some of the provided rights are not included * in the table, likely due to a bug in the mktables script. */ CAP_NONE(&zero); if (!cap_rights_contains(&zero, &diff)) fprintf(fp, ",unknown rights"); } /* * Pre-sort the set of rights, which has a partial ordering defined by the * subset relation. This lets sysdecode_cap_rights() print a list of minimal * length with a single pass over the "caprights" table. */ static void __attribute__((constructor)) sysdecode_cap_rights_init(void) { cap_rights_t tr, qr; struct name_table *t, *q, tmp; bool swapped; do { for (t = caprights, swapped = false; t->str != NULL; t++) { cap_rights_init(&tr, t->val); for (q = t + 1; q->str != NULL; q++) { cap_rights_init(&qr, q->val); if (cap_rights_contains(&qr, &tr)) { tmp = *t; *t = *q; *q = tmp; swapped = true; } } } } while (swapped); } static struct name_table cmsgtypeip[] = { X(IP_RECVDSTADDR) X(IP_RECVTTL) X(IP_RECVOPTS) X(IP_RECVRETOPTS) X(IP_RECVIF) X(IP_RECVTOS) X(IP_FLOWID) X(IP_FLOWTYPE) X(IP_RSSBUCKETID) XEND }; static struct name_table cmsgtypeipv6[] = { #if 0 /* The RFC 2292 defines are kernel space only. */ X(IPV6_2292PKTINFO) X(IPV6_2292HOPLIMIT) X(IPV6_2292HOPOPTS) X(IPV6_2292DSTOPTS) X(IPV6_2292RTHDR) X(IPV6_2292NEXTHOP) #endif X(IPV6_PKTINFO) X(IPV6_HOPLIMIT) X(IPV6_HOPOPTS) X(IPV6_DSTOPTS) X(IPV6_RTHDR) X(IPV6_NEXTHOP) X(IPV6_TCLASS) X(IPV6_FLOWID) X(IPV6_FLOWTYPE) X(IPV6_RSSBUCKETID) X(IPV6_PATHMTU) X(IPV6_RTHDRDSTOPTS) X(IPV6_USE_MIN_MTU) X(IPV6_DONTFRAG) X(IPV6_PREFER_TEMPADDR) XEND }; static struct name_table cmsgtypesctp[] = { X(SCTP_INIT) X(SCTP_SNDRCV) X(SCTP_EXTRCV) X(SCTP_SNDINFO) X(SCTP_RCVINFO) X(SCTP_NXTINFO) X(SCTP_PRINFO) X(SCTP_AUTHINFO) X(SCTP_DSTADDRV4) X(SCTP_DSTADDRV6) XEND }; const char * sysdecode_cmsg_type(int cmsg_level, int cmsg_type) { if (cmsg_level == SOL_SOCKET) return (lookup_value(cmsgtypesocket, cmsg_type)); if (cmsg_level == IPPROTO_IP) return (lookup_value(cmsgtypeip, cmsg_type)); if (cmsg_level == IPPROTO_IPV6) return (lookup_value(cmsgtypeipv6, cmsg_type)); if (cmsg_level == IPPROTO_SCTP) return (lookup_value(cmsgtypesctp, cmsg_type)); return (NULL); } const char * sysdecode_sctp_pr_policy(int policy) { return (lookup_value(sctpprpolicy, policy)); } static struct name_table sctpsndflags[] = { X(SCTP_EOF) X(SCTP_ABORT) X(SCTP_UNORDERED) X(SCTP_ADDR_OVER) X(SCTP_SENDALL) X(SCTP_EOR) X(SCTP_SACK_IMMEDIATELY) XEND }; bool sysdecode_sctp_snd_flags(FILE *fp, int flags, int *rem) { return (print_mask_int(fp, sctpsndflags, flags, rem)); } static struct name_table sctprcvflags[] = { X(SCTP_UNORDERED) XEND }; bool sysdecode_sctp_rcv_flags(FILE *fp, int flags, int *rem) { return (print_mask_int(fp, sctprcvflags, flags, rem)); } static struct name_table sctpnxtflags[] = { X(SCTP_UNORDERED) X(SCTP_COMPLETE) X(SCTP_NOTIFICATION) XEND }; bool sysdecode_sctp_nxt_flags(FILE *fp, int flags, int *rem) { return (print_mask_int(fp, sctpnxtflags, flags, rem)); } static struct name_table sctpsinfoflags[] = { X(SCTP_EOF) X(SCTP_ABORT) X(SCTP_UNORDERED) X(SCTP_ADDR_OVER) X(SCTP_SENDALL) X(SCTP_EOR) X(SCTP_SACK_IMMEDIATELY) XEND }; void sysdecode_sctp_sinfo_flags(FILE *fp, int sinfo_flags) { const char *temp; int rem; bool printed; printed = print_mask_0(fp, sctpsinfoflags, sinfo_flags, &rem); if (rem & ~SCTP_PR_SCTP_ALL) { fprintf(fp, "%s%#x", printed ? "|" : "", rem & ~SCTP_PR_SCTP_ALL); printed = true; rem &= ~SCTP_PR_SCTP_ALL; } if (rem != 0) { temp = sysdecode_sctp_pr_policy(rem); if (temp != NULL) { fprintf(fp, "%s%s", printed ? "|" : "", temp); } else { fprintf(fp, "%s%#x", printed ? "|" : "", rem); } } } bool sysdecode_shmflags(FILE *fp, int flags, int *rem) { return (print_mask_0(fp, shmflags, flags, rem)); } const char * sysdecode_itimer(int which) { return (lookup_value(itimerwhich, which)); } diff --git a/lib/libsysdecode/mktables b/lib/libsysdecode/mktables index a0ff8ee8516c..68c3450a03f7 100644 --- a/lib/libsysdecode/mktables +++ b/lib/libsysdecode/mktables @@ -1,188 +1,188 @@ #!/bin/sh # # Copyright (c) 2006 "David Kirchner" . All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # $FreeBSD$ # # Generates tables.h # # Originally this script was 'mksubr' for kdump which generated a complete # C file along with function definitions. Now this script generates tables # of constants and names extracted from header files. set -e LC_ALL=C; export LC_ALL if [ -z "$1" ] then echo "usage: sh $0 include-dir [output-file]" exit 1 fi include_dir=$1 if [ -n "$2" ]; then output_file="$2" output_tmp=$(mktemp -u) exec > "$output_tmp" fi all_headers= # # Generate a table C #definitions. The including file can define the # TABLE_NAME(n), TABLE_ENTRY(x), and TABLE_END macros to define what # the tables map to. # gen_table() { local name grep file excl filter name=$1 grep=$2 file=$3 excl=$4 if [ -z "$excl" ]; then filter="cat" else filter="egrep -v" fi cat <<_EOF_ TABLE_START(${name}) _EOF_ if [ -e "${include_dir}/${file}" ]; then all_headers="${all_headers:+${all_headers} }${file}" egrep "^#[[:space:]]*define[[:space:]]+"${grep}"[[:space:]]*" \ $include_dir/$file | ${filter} ${excl} | \ awk '{ for (i = 1; i <= NF; i++) \ if ($i ~ /define/) \ break; \ ++i; \ printf "TABLE_ENTRY(%s)\n", $i }' fi cat <<_EOF_ TABLE_END _EOF_ } cat <<_EOF_ /* This file is auto-generated. */ _EOF_ gen_table "accessmode" "[A-Z]_OK[[:space:]]+0?x?[0-9A-Fa-f]+" "sys/unistd.h" gen_table "acltype" "ACL_TYPE_[A-Z4_]+[[:space:]]+0x[0-9]+" "sys/acl.h" gen_table "atflags" "AT_[A-Z_]+[[:space:]]+0x[0-9]+" "sys/fcntl.h" gen_table "capfcntl" "CAP_FCNTL_[A-Z]+[[:space:]]+\(1" "sys/capsicum.h" gen_table "closerangeflags" "CLOSE_RANGE_[A-Z]+[[:space:]]+\([0-9]+<<[0-9]+\)" "sys/unistd.h" gen_table "extattrns" "EXTATTR_NAMESPACE_[A-Z]+[[:space:]]+0x[0-9]+" "sys/extattr.h" gen_table "fadvisebehav" "POSIX_FADV_[A-Z]+[[:space:]]+[0-9]+" "sys/fcntl.h" gen_table "openflags" "O_[A-Z]+[[:space:]]+0x[0-9A-Fa-f]+" "sys/fcntl.h" "O_RDONLY|O_RDWR|O_WRONLY" gen_table "flockops" "LOCK_[A-Z]+[[:space:]]+0x[0-9]+" "sys/fcntl.h" gen_table "kldsymcmd" "KLDSYM_[A-Z]+[[:space:]]+[0-9]+" "sys/linker.h" gen_table "kldunloadfflags" "LINKER_UNLOAD_[A-Z]+[[:space:]]+[0-9]+" "sys/linker.h" gen_table "lio_listiomodes" "LIO_(NO)?WAIT[[:space:]]+[0-9]+" "aio.h" gen_table "madvisebehav" "_?MADV_[A-Z]+[[:space:]]+[0-9]+" "sys/mman.h" gen_table "minheritflags" "INHERIT_[A-Z]+[[:space:]]+[0-9]+" "sys/mman.h" gen_table "mlockallflags" "MCL_[A-Z]+[[:space:]]+0x[0-9]+" "sys/mman.h" gen_table "mmapprot" "PROT_[A-Z]+[[:space:]]+0x[0-9A-Fa-f]+" "sys/mman.h" gen_table "ngbtsolevel" "SOL_[A-Z0-9]+[[:space:]]+0x[0-9A-Fa-f]+" "netgraph/bluetooth/include/ng_btsocket.h" gen_table "fileflags" "[SU]F_[A-Z]+[[:space:]]+0x[0-9A-Fa-f]+" "sys/stat.h" "UF_COMPRESSED|UF_TRACKED|UF_SETTABLE|SF_SETTABLE" gen_table "filemode" "S_[A-Z]+[[:space:]]+[0-6]{7}" "sys/stat.h" gen_table "keventflags" "EV_[A-Z]+[[:space:]]+0x[0-9]+" "sys/event.h" "EV_SYSFLAGS|EV_DROP|EV_FLAG[12]" gen_table "keventfilters" "EVFILT_[A-Z]+[[:space:]]+\(-[0-9]+\)" "sys/event.h" gen_table "mountflags" "MNT_[A-Z]+[[:space:]]+0x[0-9]+" "sys/mount.h" gen_table "msyncflags" "MS_[A-Z]+[[:space:]]+0x[0-9]+" "sys/mman.h" gen_table "nfssvcflags" "NFSSVC_[A-Z0-9]+[[:space:]]+0x[0-9]+" "nfs/nfssvc.h" gen_table "pathconfname" "_PC_[A-Z4_]+[[:space:]]+[0-9]+" "sys/unistd.h" gen_table "prio" "PRIO_[A-Z]+[[:space:]]+[0-9]" "sys/resource.h" gen_table "procctlcmd" "PROC_[A-Z_]+[[:space:]]+[0-9]" "sys/procctl.h" "PROC_TRACE_CTL_" gen_table "ptraceop" "PT_[[:alnum:]_]+[[:space:]]+[0-9]+" "sys/ptrace.h" gen_table "quotactlcmds" "Q_[A-Z]+[[:space:]]+0x[0-9]+" "ufs/ufs/quota.h" gen_table "rebootopt" "RB_[A-Z]+[[:space:]]+0x[0-9]+" "sys/reboot.h" gen_table "rforkflags" "RF[A-Z]+[[:space:]]+\([0-9]+<<[0-9]+\)" "sys/unistd.h" gen_table "rlimit" "RLIMIT_[A-Z]+[[:space:]]+[0-9]+" "sys/resource.h" gen_table "rusage" "RUSAGE_[A-Z]+[[:space:]]+[-0-9]+" "sys/resource.h" gen_table "schedpolicy" "SCHED_[A-Z]+[[:space:]]+[0-9]+" "sys/sched.h" gen_table "sendfileflags" "SF_[A-Z]+[[:space:]]+[0-9]+" "sys/socket.h" gen_table "shmatflags" "SHM_[A-Z]+[[:space:]]+[0-9]{6}" "sys/shm.h" gen_table "shutdownhow" "SHUT_[A-Z]+[[:space:]]+[0-9]+" "sys/socket.h" gen_table "sigbuscode" "BUS_[A-Z]+[[:space:]]+[0-9]+" "sys/signal.h" gen_table "sigchldcode" "CLD_[A-Z]+[[:space:]]+[0-9]+" "sys/signal.h" gen_table "sigfpecode" "FPE_[A-Z]+[[:space:]]+[0-9]+" "sys/signal.h" gen_table "sigprocmaskhow" "SIG_[A-Z]+[[:space:]]+[0-9]+" "sys/signal.h" gen_table "sigillcode" "ILL_[A-Z]+[[:space:]]+[0-9]+" "sys/signal.h" gen_table "sigsegvcode" "SEGV_[A-Z]+[[:space:]]+[0-9]+" "sys/signal.h" gen_table "sigtrapcode" "TRAP_[A-Z]+[[:space:]]+[0-9]+" "sys/signal.h" gen_table "sockdomain" "PF_[[:alnum:]]+[[:space:]]+" "sys/socket.h" gen_table "sockfamily" "AF_[[:alnum:]]+[[:space:]]+" "sys/socket.h" gen_table "sockipproto" "IPPROTO_[[:alnum:]]+[[:space:]]+" "netinet/in.h" gen_table "sockopt" "SO_[A-Z_]+[[:space:]]+0x[0-9]+" "sys/socket.h" gen_table "sockoptip" "(IP_[[:alnum:]_]+|MCAST_[[:alnum:]_]+_GROUP)[[:space:]]+" "netinet/in.h" "IP_DEFAULT|IP_MIN|IP_MAX|IP_PORTRANGE" gen_table "sockoptipv6" "IPV6_[[:alnum:]_]+[[:space:]]+[0-9]+" "netinet6/in6.h" "IPV6_ADDR_|IPV6_TAG_DIRECT|IPV6_OPTIONS|IPV6_RECVOPTS|IPV6_RECVRETOPTS|IPV6_RECVDSTADDR|IPV6_RETOPTS|IPV6_2292|IPV6_RECVRTHDRDSTOPTS|IPV6_REACHCONF|IPV6_PKTOPTIONS" gen_table "sockoptsctp" "SCTP_[[:alnum:]_]+[[:space:]]+[0-9]+" "netinet/sctp.h" gen_table "sockopttcp" "TCP_[[:alnum:]_]+[[:space:]]+[0-9]+" "netinet/tcp.h" "TCP_MIN|TCP_MAX[^S]|TCP_MSS|TCP_[[:alnum:]_]+_MAX|TCP_FASTOPEN_MIN_COOKIE_LEN|TCP_FASTOPEN_PSK_LEN" gen_table "sockoptudp" "UDP_[[:alnum:]]+[[:space:]]+[0-9]+" "netinet/udp.h" "UDP_ENCAP_" gen_table "sockoptudplite" "UDPLITE_[[:alnum:]_]+[[:space:]]+[0-9]+" "netinet/udplite.h" gen_table "socktype" "SOCK_[A-Z]+[[:space:]]+[1-9]+[0-9]*" "sys/socket.h" gen_table "thrcreateflags" "THR_[A-Z]+[[:space:]]+0x[0-9]+" "sys/thr.h" gen_table "umtxop" "UMTX_OP_[[:alnum:]][[:alnum:]_]*[[:space:]]+[0-9]+" "sys/umtx.h" gen_table "umtxopflags" "UMTX_OP__[[:alnum:]_]+[[:space:]]+[0-9]+" "sys/umtx.h" gen_table "vmprot" "VM_PROT_[A-Z_]+[[:space:]]+\(\(vm_prot_t\)[[:space:]]+0x[0-9]+\)" "vm/vm.h" gen_table "vmresult" "KERN_[A-Z_]+[[:space:]]+[0-9]+" "vm/vm_param.h" gen_table "wait6opt" "W[A-Z]+[[:space:]]+[0-9]+" "sys/wait.h" gen_table "seekwhence" "SEEK_[A-Z]+[[:space:]]+[0-9]+" "sys/unistd.h" gen_table "fcntlcmd" "F_[A-Z0-9_]+[[:space:]]+[0-9]+[[:space:]]+" "sys/fcntl.h" "F_CANCEL|F_..LCK" -gen_table "mmapflags" "MAP_[A-Z_]+[[:space:]]+0x[0-9A-Fa-f]+" "sys/mman.h" +gen_table "mmapflags" "MAP_[2-3A-Z_]+[[:space:]]+0x[0-9A-Fa-f]+" "sys/mman.h" gen_table "rtpriofuncs" "RTP_[A-Z]+[[:space:]]+[0-9]+" "sys/rtprio.h" gen_table "msgflags" "MSG_[A-Z_]+[[:space:]]+0x[0-9]+" "sys/socket.h" "MSG_SOCALLBCK|MSG_MORETOCOME|MSG_TLSAPPDATA" gen_table "sigcode" "SI_[A-Z]+[[:space:]]+0(x[0-9abcdef]+)?" "sys/signal.h" gen_table "umtxcvwaitflags" "CVWAIT_[A-Z_]+[[:space:]]+0x[0-9]+" "sys/umtx.h" gen_table "umtxrwlockflags" "URWLOCK_PREFER_READER[[:space:]]+0x[0-9]+" "sys/umtx.h" gen_table "caprights" "CAP_[A-Z_]+[[:space:]]+((CAPRIGHT\([0-9],[[:space:]]+0x[0-9]{16}ULL\))|(\(CAP_[A-Z_]+[[:space:]]*\|.*\)))" "sys/capsicum.h" gen_table "sctpprpolicy" "SCTP_PR_SCTP_[A-Z_]+[[:space:]]+0x[0-9]+" "netinet/sctp_uio.h" "SCTP_PR_SCTP_ALL" gen_table "cmsgtypesocket" "SCM_[A-Z_]+[[:space:]]+0x[0-9]+" "sys/socket.h" if [ -e "${include_dir}/x86/sysarch.h" ]; then gen_table "sysarchnum" "(AMD64|I386)_[A-Z86_]+[[:space:]]+[0-9]+" "x86/sysarch.h" else gen_table "sysarchnum" "[A-Z_]+[[:space:]]+[0-9]+" "machine/sysarch.h" fi gen_table "shmflags" "SHM_[A-Z_]+[[:space:]]+0x[0-9]+" "sys/mman.h" "SHM_ANON" gen_table "itimerwhich" "ITIMER_[A-Z]+[[:space:]]+[0-9]+" "sys/time.h" # Generate a .depend file for our output file if [ -n "$output_file" ]; then depend_tmp=$(mktemp -u) { echo "$output_file: \\" echo "$all_headers" | tr ' ' '\n' | sort -u | sed -e "s,^, $include_dir/," -e 's,$, \\,' echo } > "$depend_tmp" if cmp -s "$output_tmp" "$output_file"; then rm -f "$output_tmp" "$depend_tmp" else mv -f "$depend_tmp" ".depend.${output_file}" mv -f "$output_tmp" "$output_file" fi fi diff --git a/sys/kern/uipc_shm.c b/sys/kern/uipc_shm.c index fb210248f9ac..be4a700c4944 100644 --- a/sys/kern/uipc_shm.c +++ b/sys/kern/uipc_shm.c @@ -1,2196 +1,2190 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2006, 2011, 2016-2017 Robert N. M. Watson * Copyright 2020 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by BAE Systems, the University of * Cambridge Computer Laboratory, and Memorial University under DARPA/AFRL * contract FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent * Computing (TC) research program. * * Portions of this software were developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Support for shared swap-backed anonymous memory objects via * shm_open(2), shm_rename(2), and shm_unlink(2). * While most of the implementation is here, vm_mmap.c contains * mapping logic changes. * * posixshmcontrol(1) allows users to inspect the state of the memory * objects. Per-uid swap resource limit controls total amount of * memory that user can consume for anonymous objects, including * shared. */ #include __FBSDID("$FreeBSD$"); #include "opt_capsicum.h" #include "opt_ktrace.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include struct shm_mapping { char *sm_path; Fnv32_t sm_fnv; struct shmfd *sm_shmfd; LIST_ENTRY(shm_mapping) sm_link; }; static MALLOC_DEFINE(M_SHMFD, "shmfd", "shared memory file descriptor"); static LIST_HEAD(, shm_mapping) *shm_dictionary; static struct sx shm_dict_lock; static struct mtx shm_timestamp_lock; static u_long shm_hash; static struct unrhdr64 shm_ino_unr; static dev_t shm_dev_ino; #define SHM_HASH(fnv) (&shm_dictionary[(fnv) & shm_hash]) static void shm_init(void *arg); static void shm_insert(char *path, Fnv32_t fnv, struct shmfd *shmfd); static struct shmfd *shm_lookup(char *path, Fnv32_t fnv); static int shm_remove(char *path, Fnv32_t fnv, struct ucred *ucred); static void shm_doremove(struct shm_mapping *map); static int shm_dotruncate_cookie(struct shmfd *shmfd, off_t length, void *rl_cookie); static int shm_dotruncate_locked(struct shmfd *shmfd, off_t length, void *rl_cookie); static int shm_copyin_path(struct thread *td, const char *userpath_in, char **path_out); static int shm_deallocate(struct shmfd *shmfd, off_t *offset, off_t *length, int flags); static fo_rdwr_t shm_read; static fo_rdwr_t shm_write; static fo_truncate_t shm_truncate; static fo_ioctl_t shm_ioctl; static fo_stat_t shm_stat; static fo_close_t shm_close; static fo_chmod_t shm_chmod; static fo_chown_t shm_chown; static fo_seek_t shm_seek; static fo_fill_kinfo_t shm_fill_kinfo; static fo_mmap_t shm_mmap; static fo_get_seals_t shm_get_seals; static fo_add_seals_t shm_add_seals; static fo_fallocate_t shm_fallocate; static fo_fspacectl_t shm_fspacectl; /* File descriptor operations. */ struct fileops shm_ops = { .fo_read = shm_read, .fo_write = shm_write, .fo_truncate = shm_truncate, .fo_ioctl = shm_ioctl, .fo_poll = invfo_poll, .fo_kqfilter = invfo_kqfilter, .fo_stat = shm_stat, .fo_close = shm_close, .fo_chmod = shm_chmod, .fo_chown = shm_chown, .fo_sendfile = vn_sendfile, .fo_seek = shm_seek, .fo_fill_kinfo = shm_fill_kinfo, .fo_mmap = shm_mmap, .fo_get_seals = shm_get_seals, .fo_add_seals = shm_add_seals, .fo_fallocate = shm_fallocate, .fo_fspacectl = shm_fspacectl, .fo_flags = DFLAG_PASSABLE | DFLAG_SEEKABLE, }; FEATURE(posix_shm, "POSIX shared memory"); static SYSCTL_NODE(_vm, OID_AUTO, largepages, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); static int largepage_reclaim_tries = 1; SYSCTL_INT(_vm_largepages, OID_AUTO, reclaim_tries, CTLFLAG_RWTUN, &largepage_reclaim_tries, 0, "Number of contig reclaims before giving up for default alloc policy"); static int uiomove_object_page(vm_object_t obj, size_t len, struct uio *uio) { vm_page_t m; vm_pindex_t idx; size_t tlen; int error, offset, rv; idx = OFF_TO_IDX(uio->uio_offset); offset = uio->uio_offset & PAGE_MASK; tlen = MIN(PAGE_SIZE - offset, len); rv = vm_page_grab_valid_unlocked(&m, obj, idx, VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY | VM_ALLOC_NOCREAT); if (rv == VM_PAGER_OK) goto found; /* * Read I/O without either a corresponding resident page or swap * page: use zero_region. This is intended to avoid instantiating * pages on read from a sparse region. */ VM_OBJECT_WLOCK(obj); m = vm_page_lookup(obj, idx); if (uio->uio_rw == UIO_READ && m == NULL && !vm_pager_has_page(obj, idx, NULL, NULL)) { VM_OBJECT_WUNLOCK(obj); return (uiomove(__DECONST(void *, zero_region), tlen, uio)); } /* * Although the tmpfs vnode lock is held here, it is * nonetheless safe to sleep waiting for a free page. The * pageout daemon does not need to acquire the tmpfs vnode * lock to page out tobj's pages because tobj is a OBJT_SWAP * type object. */ rv = vm_page_grab_valid(&m, obj, idx, VM_ALLOC_NORMAL | VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY); if (rv != VM_PAGER_OK) { VM_OBJECT_WUNLOCK(obj); if (bootverbose) { printf("uiomove_object: vm_obj %p idx %jd " "pager error %d\n", obj, idx, rv); } return (rv == VM_PAGER_AGAIN ? ENOSPC : EIO); } VM_OBJECT_WUNLOCK(obj); found: error = uiomove_fromphys(&m, offset, tlen, uio); if (uio->uio_rw == UIO_WRITE && error == 0) vm_page_set_dirty(m); vm_page_activate(m); vm_page_sunbusy(m); return (error); } int uiomove_object(vm_object_t obj, off_t obj_size, struct uio *uio) { ssize_t resid; size_t len; int error; error = 0; while ((resid = uio->uio_resid) > 0) { if (obj_size <= uio->uio_offset) break; len = MIN(obj_size - uio->uio_offset, resid); if (len == 0) break; error = uiomove_object_page(obj, len, uio); if (error != 0 || resid == uio->uio_resid) break; } return (error); } static u_long count_largepages[MAXPAGESIZES]; static int shm_largepage_phys_populate(vm_object_t object, vm_pindex_t pidx, int fault_type, vm_prot_t max_prot, vm_pindex_t *first, vm_pindex_t *last) { vm_page_t m __diagused; int psind; psind = object->un_pager.phys.data_val; if (psind == 0 || pidx >= object->size) return (VM_PAGER_FAIL); *first = rounddown2(pidx, pagesizes[psind] / PAGE_SIZE); /* * We only busy the first page in the superpage run. It is * useless to busy whole run since we only remove full * superpage, and it takes too long to busy e.g. 512 * 512 == * 262144 pages constituing 1G amd64 superage. */ m = vm_page_grab(object, *first, VM_ALLOC_NORMAL | VM_ALLOC_NOCREAT); MPASS(m != NULL); *last = *first + atop(pagesizes[psind]) - 1; return (VM_PAGER_OK); } static boolean_t shm_largepage_phys_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after) { int psind; psind = object->un_pager.phys.data_val; if (psind == 0 || pindex >= object->size) return (FALSE); if (before != NULL) { *before = pindex - rounddown2(pindex, pagesizes[psind] / PAGE_SIZE); } if (after != NULL) { *after = roundup2(pindex, pagesizes[psind] / PAGE_SIZE) - pindex; } return (TRUE); } static void shm_largepage_phys_ctor(vm_object_t object, vm_prot_t prot, vm_ooffset_t foff, struct ucred *cred) { } static void shm_largepage_phys_dtor(vm_object_t object) { int psind; psind = object->un_pager.phys.data_val; if (psind != 0) { atomic_subtract_long(&count_largepages[psind], object->size / (pagesizes[psind] / PAGE_SIZE)); vm_wire_sub(object->size); } else { KASSERT(object->size == 0, ("largepage phys obj %p not initialized bit size %#jx > 0", object, (uintmax_t)object->size)); } } static const struct phys_pager_ops shm_largepage_phys_ops = { .phys_pg_populate = shm_largepage_phys_populate, .phys_pg_haspage = shm_largepage_phys_haspage, .phys_pg_ctor = shm_largepage_phys_ctor, .phys_pg_dtor = shm_largepage_phys_dtor, }; bool shm_largepage(struct shmfd *shmfd) { return (shmfd->shm_object->type == OBJT_PHYS); } static void shm_pager_freespace(vm_object_t obj, vm_pindex_t start, vm_size_t size) { struct shmfd *shm; vm_size_t c; swap_pager_freespace(obj, start, size, &c); if (c == 0) return; shm = obj->un_pager.swp.swp_priv; if (shm == NULL) return; KASSERT(shm->shm_pages >= c, ("shm %p pages %jd free %jd", shm, (uintmax_t)shm->shm_pages, (uintmax_t)c)); shm->shm_pages -= c; } static void shm_page_inserted(vm_object_t obj, vm_page_t m) { struct shmfd *shm; shm = obj->un_pager.swp.swp_priv; if (shm == NULL) return; if (!vm_pager_has_page(obj, m->pindex, NULL, NULL)) shm->shm_pages += 1; } static void shm_page_removed(vm_object_t obj, vm_page_t m) { struct shmfd *shm; shm = obj->un_pager.swp.swp_priv; if (shm == NULL) return; if (!vm_pager_has_page(obj, m->pindex, NULL, NULL)) { KASSERT(shm->shm_pages >= 1, ("shm %p pages %jd free 1", shm, (uintmax_t)shm->shm_pages)); shm->shm_pages -= 1; } } static struct pagerops shm_swap_pager_ops = { .pgo_kvme_type = KVME_TYPE_SWAP, .pgo_freespace = shm_pager_freespace, .pgo_page_inserted = shm_page_inserted, .pgo_page_removed = shm_page_removed, }; static int shmfd_pager_type = -1; static int shm_seek(struct file *fp, off_t offset, int whence, struct thread *td) { struct shmfd *shmfd; off_t foffset; int error; shmfd = fp->f_data; foffset = foffset_lock(fp, 0); error = 0; switch (whence) { case L_INCR: if (foffset < 0 || (offset > 0 && foffset > OFF_MAX - offset)) { error = EOVERFLOW; break; } offset += foffset; break; case L_XTND: if (offset > 0 && shmfd->shm_size > OFF_MAX - offset) { error = EOVERFLOW; break; } offset += shmfd->shm_size; break; case L_SET: break; default: error = EINVAL; } if (error == 0) { if (offset < 0 || offset > shmfd->shm_size) error = EINVAL; else td->td_uretoff.tdu_off = offset; } foffset_unlock(fp, offset, error != 0 ? FOF_NOUPDATE : 0); return (error); } static int shm_read(struct file *fp, struct uio *uio, struct ucred *active_cred, int flags, struct thread *td) { struct shmfd *shmfd; void *rl_cookie; int error; shmfd = fp->f_data; #ifdef MAC error = mac_posixshm_check_read(active_cred, fp->f_cred, shmfd); if (error) return (error); #endif foffset_lock_uio(fp, uio, flags); rl_cookie = rangelock_rlock(&shmfd->shm_rl, uio->uio_offset, uio->uio_offset + uio->uio_resid, &shmfd->shm_mtx); error = uiomove_object(shmfd->shm_object, shmfd->shm_size, uio); rangelock_unlock(&shmfd->shm_rl, rl_cookie, &shmfd->shm_mtx); foffset_unlock_uio(fp, uio, flags); return (error); } static int shm_write(struct file *fp, struct uio *uio, struct ucred *active_cred, int flags, struct thread *td) { struct shmfd *shmfd; void *rl_cookie; int error; off_t size; shmfd = fp->f_data; #ifdef MAC error = mac_posixshm_check_write(active_cred, fp->f_cred, shmfd); if (error) return (error); #endif if (shm_largepage(shmfd) && shmfd->shm_lp_psind == 0) return (EINVAL); foffset_lock_uio(fp, uio, flags); if (uio->uio_resid > OFF_MAX - uio->uio_offset) { /* * Overflow is only an error if we're supposed to expand on * write. Otherwise, we'll just truncate the write to the * size of the file, which can only grow up to OFF_MAX. */ if ((shmfd->shm_flags & SHM_GROW_ON_WRITE) != 0) { foffset_unlock_uio(fp, uio, flags); return (EFBIG); } size = shmfd->shm_size; } else { size = uio->uio_offset + uio->uio_resid; } if ((flags & FOF_OFFSET) == 0) { rl_cookie = rangelock_wlock(&shmfd->shm_rl, 0, OFF_MAX, &shmfd->shm_mtx); } else { rl_cookie = rangelock_wlock(&shmfd->shm_rl, uio->uio_offset, size, &shmfd->shm_mtx); } if ((shmfd->shm_seals & F_SEAL_WRITE) != 0) { error = EPERM; } else { error = 0; if ((shmfd->shm_flags & SHM_GROW_ON_WRITE) != 0 && size > shmfd->shm_size) { error = shm_dotruncate_cookie(shmfd, size, rl_cookie); } if (error == 0) error = uiomove_object(shmfd->shm_object, shmfd->shm_size, uio); } rangelock_unlock(&shmfd->shm_rl, rl_cookie, &shmfd->shm_mtx); foffset_unlock_uio(fp, uio, flags); return (error); } static int shm_truncate(struct file *fp, off_t length, struct ucred *active_cred, struct thread *td) { struct shmfd *shmfd; #ifdef MAC int error; #endif shmfd = fp->f_data; #ifdef MAC error = mac_posixshm_check_truncate(active_cred, fp->f_cred, shmfd); if (error) return (error); #endif return (shm_dotruncate(shmfd, length)); } int shm_ioctl(struct file *fp, u_long com, void *data, struct ucred *active_cred, struct thread *td) { struct shmfd *shmfd; struct shm_largepage_conf *conf; void *rl_cookie; shmfd = fp->f_data; switch (com) { case FIONBIO: case FIOASYNC: /* * Allow fcntl(fd, F_SETFL, O_NONBLOCK) to work, * just like it would on an unlinked regular file */ return (0); case FIOSSHMLPGCNF: if (!shm_largepage(shmfd)) return (ENOTTY); conf = data; if (shmfd->shm_lp_psind != 0 && conf->psind != shmfd->shm_lp_psind) return (EINVAL); if (conf->psind <= 0 || conf->psind >= MAXPAGESIZES || pagesizes[conf->psind] == 0) return (EINVAL); if (conf->alloc_policy != SHM_LARGEPAGE_ALLOC_DEFAULT && conf->alloc_policy != SHM_LARGEPAGE_ALLOC_NOWAIT && conf->alloc_policy != SHM_LARGEPAGE_ALLOC_HARD) return (EINVAL); rl_cookie = rangelock_wlock(&shmfd->shm_rl, 0, OFF_MAX, &shmfd->shm_mtx); shmfd->shm_lp_psind = conf->psind; shmfd->shm_lp_alloc_policy = conf->alloc_policy; shmfd->shm_object->un_pager.phys.data_val = conf->psind; rangelock_unlock(&shmfd->shm_rl, rl_cookie, &shmfd->shm_mtx); return (0); case FIOGSHMLPGCNF: if (!shm_largepage(shmfd)) return (ENOTTY); conf = data; rl_cookie = rangelock_rlock(&shmfd->shm_rl, 0, OFF_MAX, &shmfd->shm_mtx); conf->psind = shmfd->shm_lp_psind; conf->alloc_policy = shmfd->shm_lp_alloc_policy; rangelock_unlock(&shmfd->shm_rl, rl_cookie, &shmfd->shm_mtx); return (0); default: return (ENOTTY); } } static int shm_stat(struct file *fp, struct stat *sb, struct ucred *active_cred) { struct shmfd *shmfd; #ifdef MAC int error; #endif shmfd = fp->f_data; #ifdef MAC error = mac_posixshm_check_stat(active_cred, fp->f_cred, shmfd); if (error) return (error); #endif /* * Attempt to return sanish values for fstat() on a memory file * descriptor. */ bzero(sb, sizeof(*sb)); sb->st_blksize = PAGE_SIZE; sb->st_size = shmfd->shm_size; mtx_lock(&shm_timestamp_lock); sb->st_atim = shmfd->shm_atime; sb->st_ctim = shmfd->shm_ctime; sb->st_mtim = shmfd->shm_mtime; sb->st_birthtim = shmfd->shm_birthtime; sb->st_mode = S_IFREG | shmfd->shm_mode; /* XXX */ sb->st_uid = shmfd->shm_uid; sb->st_gid = shmfd->shm_gid; mtx_unlock(&shm_timestamp_lock); sb->st_dev = shm_dev_ino; sb->st_ino = shmfd->shm_ino; sb->st_nlink = shmfd->shm_object->ref_count; if (shm_largepage(shmfd)) { sb->st_blocks = shmfd->shm_object->size / (pagesizes[shmfd->shm_lp_psind] >> PAGE_SHIFT); } else { sb->st_blocks = shmfd->shm_pages; } return (0); } static int shm_close(struct file *fp, struct thread *td) { struct shmfd *shmfd; shmfd = fp->f_data; fp->f_data = NULL; shm_drop(shmfd); return (0); } static int shm_copyin_path(struct thread *td, const char *userpath_in, char **path_out) { int error; char *path; const char *pr_path; size_t pr_pathlen; path = malloc(MAXPATHLEN, M_SHMFD, M_WAITOK); pr_path = td->td_ucred->cr_prison->pr_path; /* Construct a full pathname for jailed callers. */ pr_pathlen = strcmp(pr_path, "/") == 0 ? 0 : strlcpy(path, pr_path, MAXPATHLEN); error = copyinstr(userpath_in, path + pr_pathlen, MAXPATHLEN - pr_pathlen, NULL); if (error != 0) goto out; #ifdef KTRACE if (KTRPOINT(curthread, KTR_NAMEI)) ktrnamei(path); #endif /* Require paths to start with a '/' character. */ if (path[pr_pathlen] != '/') { error = EINVAL; goto out; } *path_out = path; out: if (error != 0) free(path, M_SHMFD); return (error); } static int shm_partial_page_invalidate(vm_object_t object, vm_pindex_t idx, int base, int end) { vm_page_t m; int rv; VM_OBJECT_ASSERT_WLOCKED(object); KASSERT(base >= 0, ("%s: base %d", __func__, base)); KASSERT(end - base <= PAGE_SIZE, ("%s: base %d end %d", __func__, base, end)); retry: m = vm_page_grab(object, idx, VM_ALLOC_NOCREAT); if (m != NULL) { MPASS(vm_page_all_valid(m)); } else if (vm_pager_has_page(object, idx, NULL, NULL)) { m = vm_page_alloc(object, idx, VM_ALLOC_NORMAL | VM_ALLOC_WAITFAIL); if (m == NULL) goto retry; vm_object_pip_add(object, 1); VM_OBJECT_WUNLOCK(object); rv = vm_pager_get_pages(object, &m, 1, NULL, NULL); VM_OBJECT_WLOCK(object); vm_object_pip_wakeup(object); if (rv == VM_PAGER_OK) { /* * Since the page was not resident, and therefore not * recently accessed, immediately enqueue it for * asynchronous laundering. The current operation is * not regarded as an access. */ vm_page_launder(m); } else { vm_page_free(m); VM_OBJECT_WUNLOCK(object); return (EIO); } } if (m != NULL) { pmap_zero_page_area(m, base, end - base); KASSERT(vm_page_all_valid(m), ("%s: page %p is invalid", __func__, m)); vm_page_set_dirty(m); vm_page_xunbusy(m); } return (0); } static int shm_dotruncate_locked(struct shmfd *shmfd, off_t length, void *rl_cookie) { vm_object_t object; vm_pindex_t nobjsize; vm_ooffset_t delta; int base, error; KASSERT(length >= 0, ("shm_dotruncate: length < 0")); object = shmfd->shm_object; VM_OBJECT_ASSERT_WLOCKED(object); rangelock_cookie_assert(rl_cookie, RA_WLOCKED); if (length == shmfd->shm_size) return (0); nobjsize = OFF_TO_IDX(length + PAGE_MASK); /* Are we shrinking? If so, trim the end. */ if (length < shmfd->shm_size) { if ((shmfd->shm_seals & F_SEAL_SHRINK) != 0) return (EPERM); /* * Disallow any requests to shrink the size if this * object is mapped into the kernel. */ if (shmfd->shm_kmappings > 0) return (EBUSY); /* * Zero the truncated part of the last page. */ base = length & PAGE_MASK; if (base != 0) { error = shm_partial_page_invalidate(object, OFF_TO_IDX(length), base, PAGE_SIZE); if (error) return (error); } delta = IDX_TO_OFF(object->size - nobjsize); if (nobjsize < object->size) vm_object_page_remove(object, nobjsize, object->size, 0); /* Free the swap accounted for shm */ swap_release_by_cred(delta, object->cred); object->charge -= delta; } else { if ((shmfd->shm_seals & F_SEAL_GROW) != 0) return (EPERM); /* Try to reserve additional swap space. */ delta = IDX_TO_OFF(nobjsize - object->size); if (!swap_reserve_by_cred(delta, object->cred)) return (ENOMEM); object->charge += delta; } shmfd->shm_size = length; mtx_lock(&shm_timestamp_lock); vfs_timestamp(&shmfd->shm_ctime); shmfd->shm_mtime = shmfd->shm_ctime; mtx_unlock(&shm_timestamp_lock); object->size = nobjsize; return (0); } static int shm_dotruncate_largepage(struct shmfd *shmfd, off_t length, void *rl_cookie) { vm_object_t object; vm_page_t m; vm_pindex_t newobjsz; vm_pindex_t oldobjsz __unused; int aflags, error, i, psind, try; KASSERT(length >= 0, ("shm_dotruncate: length < 0")); object = shmfd->shm_object; VM_OBJECT_ASSERT_WLOCKED(object); rangelock_cookie_assert(rl_cookie, RA_WLOCKED); oldobjsz = object->size; newobjsz = OFF_TO_IDX(length); if (length == shmfd->shm_size) return (0); psind = shmfd->shm_lp_psind; if (psind == 0 && length != 0) return (EINVAL); if ((length & (pagesizes[psind] - 1)) != 0) return (EINVAL); if (length < shmfd->shm_size) { if ((shmfd->shm_seals & F_SEAL_SHRINK) != 0) return (EPERM); if (shmfd->shm_kmappings > 0) return (EBUSY); return (ENOTSUP); /* Pages are unmanaged. */ #if 0 vm_object_page_remove(object, newobjsz, oldobjsz, 0); object->size = newobjsz; shmfd->shm_size = length; return (0); #endif } if ((shmfd->shm_seals & F_SEAL_GROW) != 0) return (EPERM); aflags = VM_ALLOC_NORMAL | VM_ALLOC_ZERO; if (shmfd->shm_lp_alloc_policy == SHM_LARGEPAGE_ALLOC_NOWAIT) aflags |= VM_ALLOC_WAITFAIL; try = 0; /* * Extend shmfd and object, keeping all already fully * allocated large pages intact even on error, because dropped * object lock might allowed mapping of them. */ while (object->size < newobjsz) { m = vm_page_alloc_contig(object, object->size, aflags, pagesizes[psind] / PAGE_SIZE, 0, ~0, pagesizes[psind], 0, VM_MEMATTR_DEFAULT); if (m == NULL) { VM_OBJECT_WUNLOCK(object); if (shmfd->shm_lp_alloc_policy == SHM_LARGEPAGE_ALLOC_NOWAIT || (shmfd->shm_lp_alloc_policy == SHM_LARGEPAGE_ALLOC_DEFAULT && try >= largepage_reclaim_tries)) { VM_OBJECT_WLOCK(object); return (ENOMEM); } error = vm_page_reclaim_contig(aflags, pagesizes[psind] / PAGE_SIZE, 0, ~0, pagesizes[psind], 0) ? 0 : vm_wait_intr(object); if (error != 0) { VM_OBJECT_WLOCK(object); return (error); } try++; VM_OBJECT_WLOCK(object); continue; } try = 0; for (i = 0; i < pagesizes[psind] / PAGE_SIZE; i++) { if ((m[i].flags & PG_ZERO) == 0) pmap_zero_page(&m[i]); vm_page_valid(&m[i]); vm_page_xunbusy(&m[i]); } object->size += OFF_TO_IDX(pagesizes[psind]); shmfd->shm_size += pagesizes[psind]; atomic_add_long(&count_largepages[psind], 1); vm_wire_add(atop(pagesizes[psind])); } return (0); } static int shm_dotruncate_cookie(struct shmfd *shmfd, off_t length, void *rl_cookie) { int error; VM_OBJECT_WLOCK(shmfd->shm_object); error = shm_largepage(shmfd) ? shm_dotruncate_largepage(shmfd, length, rl_cookie) : shm_dotruncate_locked(shmfd, length, rl_cookie); VM_OBJECT_WUNLOCK(shmfd->shm_object); return (error); } int shm_dotruncate(struct shmfd *shmfd, off_t length) { void *rl_cookie; int error; rl_cookie = rangelock_wlock(&shmfd->shm_rl, 0, OFF_MAX, &shmfd->shm_mtx); error = shm_dotruncate_cookie(shmfd, length, rl_cookie); rangelock_unlock(&shmfd->shm_rl, rl_cookie, &shmfd->shm_mtx); return (error); } /* * shmfd object management including creation and reference counting * routines. */ struct shmfd * shm_alloc(struct ucred *ucred, mode_t mode, bool largepage) { struct shmfd *shmfd; vm_object_t obj; shmfd = malloc(sizeof(*shmfd), M_SHMFD, M_WAITOK | M_ZERO); shmfd->shm_size = 0; shmfd->shm_uid = ucred->cr_uid; shmfd->shm_gid = ucred->cr_gid; shmfd->shm_mode = mode; if (largepage) { shmfd->shm_object = phys_pager_allocate(NULL, &shm_largepage_phys_ops, NULL, shmfd->shm_size, VM_PROT_DEFAULT, 0, ucred); shmfd->shm_lp_alloc_policy = SHM_LARGEPAGE_ALLOC_DEFAULT; } else { obj = vm_pager_allocate(shmfd_pager_type, NULL, shmfd->shm_size, VM_PROT_DEFAULT, 0, ucred); VM_OBJECT_WLOCK(obj); obj->un_pager.swp.swp_priv = shmfd; VM_OBJECT_WUNLOCK(obj); shmfd->shm_object = obj; } KASSERT(shmfd->shm_object != NULL, ("shm_create: vm_pager_allocate")); vfs_timestamp(&shmfd->shm_birthtime); shmfd->shm_atime = shmfd->shm_mtime = shmfd->shm_ctime = shmfd->shm_birthtime; shmfd->shm_ino = alloc_unr64(&shm_ino_unr); refcount_init(&shmfd->shm_refs, 1); mtx_init(&shmfd->shm_mtx, "shmrl", NULL, MTX_DEF); rangelock_init(&shmfd->shm_rl); #ifdef MAC mac_posixshm_init(shmfd); mac_posixshm_create(ucred, shmfd); #endif return (shmfd); } struct shmfd * shm_hold(struct shmfd *shmfd) { refcount_acquire(&shmfd->shm_refs); return (shmfd); } void shm_drop(struct shmfd *shmfd) { vm_object_t obj; if (refcount_release(&shmfd->shm_refs)) { #ifdef MAC mac_posixshm_destroy(shmfd); #endif rangelock_destroy(&shmfd->shm_rl); mtx_destroy(&shmfd->shm_mtx); obj = shmfd->shm_object; if (!shm_largepage(shmfd)) { VM_OBJECT_WLOCK(obj); obj->un_pager.swp.swp_priv = NULL; VM_OBJECT_WUNLOCK(obj); } vm_object_deallocate(obj); free(shmfd, M_SHMFD); } } /* * Determine if the credentials have sufficient permissions for a * specified combination of FREAD and FWRITE. */ int shm_access(struct shmfd *shmfd, struct ucred *ucred, int flags) { accmode_t accmode; int error; accmode = 0; if (flags & FREAD) accmode |= VREAD; if (flags & FWRITE) accmode |= VWRITE; mtx_lock(&shm_timestamp_lock); error = vaccess(VREG, shmfd->shm_mode, shmfd->shm_uid, shmfd->shm_gid, accmode, ucred); mtx_unlock(&shm_timestamp_lock); return (error); } static void shm_init(void *arg) { char name[32]; int i; mtx_init(&shm_timestamp_lock, "shm timestamps", NULL, MTX_DEF); sx_init(&shm_dict_lock, "shm dictionary"); shm_dictionary = hashinit(1024, M_SHMFD, &shm_hash); new_unrhdr64(&shm_ino_unr, 1); shm_dev_ino = devfs_alloc_cdp_inode(); KASSERT(shm_dev_ino > 0, ("shm dev inode not initialized")); shmfd_pager_type = vm_pager_alloc_dyn_type(&shm_swap_pager_ops, OBJT_SWAP); MPASS(shmfd_pager_type != -1); for (i = 1; i < MAXPAGESIZES; i++) { if (pagesizes[i] == 0) break; #define M (1024 * 1024) #define G (1024 * M) if (pagesizes[i] >= G) snprintf(name, sizeof(name), "%luG", pagesizes[i] / G); else if (pagesizes[i] >= M) snprintf(name, sizeof(name), "%luM", pagesizes[i] / M); else snprintf(name, sizeof(name), "%lu", pagesizes[i]); #undef G #undef M SYSCTL_ADD_ULONG(NULL, SYSCTL_STATIC_CHILDREN(_vm_largepages), OID_AUTO, name, CTLFLAG_RD, &count_largepages[i], "number of non-transient largepages allocated"); } } SYSINIT(shm_init, SI_SUB_SYSV_SHM, SI_ORDER_ANY, shm_init, NULL); /* * Remove all shared memory objects that belong to a prison. */ void shm_remove_prison(struct prison *pr) { struct shm_mapping *shmm, *tshmm; u_long i; sx_xlock(&shm_dict_lock); for (i = 0; i < shm_hash + 1; i++) { LIST_FOREACH_SAFE(shmm, &shm_dictionary[i], sm_link, tshmm) { if (shmm->sm_shmfd->shm_object->cred && shmm->sm_shmfd->shm_object->cred->cr_prison == pr) shm_doremove(shmm); } } sx_xunlock(&shm_dict_lock); } /* * Dictionary management. We maintain an in-kernel dictionary to map * paths to shmfd objects. We use the FNV hash on the path to store * the mappings in a hash table. */ static struct shmfd * shm_lookup(char *path, Fnv32_t fnv) { struct shm_mapping *map; LIST_FOREACH(map, SHM_HASH(fnv), sm_link) { if (map->sm_fnv != fnv) continue; if (strcmp(map->sm_path, path) == 0) return (map->sm_shmfd); } return (NULL); } static void shm_insert(char *path, Fnv32_t fnv, struct shmfd *shmfd) { struct shm_mapping *map; map = malloc(sizeof(struct shm_mapping), M_SHMFD, M_WAITOK); map->sm_path = path; map->sm_fnv = fnv; map->sm_shmfd = shm_hold(shmfd); shmfd->shm_path = path; LIST_INSERT_HEAD(SHM_HASH(fnv), map, sm_link); } static int shm_remove(char *path, Fnv32_t fnv, struct ucred *ucred) { struct shm_mapping *map; int error; LIST_FOREACH(map, SHM_HASH(fnv), sm_link) { if (map->sm_fnv != fnv) continue; if (strcmp(map->sm_path, path) == 0) { #ifdef MAC error = mac_posixshm_check_unlink(ucred, map->sm_shmfd); if (error) return (error); #endif error = shm_access(map->sm_shmfd, ucred, FREAD | FWRITE); if (error) return (error); shm_doremove(map); return (0); } } return (ENOENT); } static void shm_doremove(struct shm_mapping *map) { map->sm_shmfd->shm_path = NULL; LIST_REMOVE(map, sm_link); shm_drop(map->sm_shmfd); free(map->sm_path, M_SHMFD); free(map, M_SHMFD); } int kern_shm_open2(struct thread *td, const char *userpath, int flags, mode_t mode, int shmflags, struct filecaps *fcaps, const char *name __unused) { struct pwddesc *pdp; struct shmfd *shmfd; struct file *fp; char *path; void *rl_cookie; Fnv32_t fnv; mode_t cmode; int error, fd, initial_seals; bool largepage; if ((shmflags & ~(SHM_ALLOW_SEALING | SHM_GROW_ON_WRITE | SHM_LARGEPAGE)) != 0) return (EINVAL); initial_seals = F_SEAL_SEAL; if ((shmflags & SHM_ALLOW_SEALING) != 0) initial_seals &= ~F_SEAL_SEAL; #ifdef CAPABILITY_MODE /* * shm_open(2) is only allowed for anonymous objects. */ if (IN_CAPABILITY_MODE(td) && (userpath != SHM_ANON)) return (ECAPMODE); #endif AUDIT_ARG_FFLAGS(flags); AUDIT_ARG_MODE(mode); if ((flags & O_ACCMODE) != O_RDONLY && (flags & O_ACCMODE) != O_RDWR) return (EINVAL); if ((flags & ~(O_ACCMODE | O_CREAT | O_EXCL | O_TRUNC | O_CLOEXEC)) != 0) return (EINVAL); largepage = (shmflags & SHM_LARGEPAGE) != 0; if (largepage && !PMAP_HAS_LARGEPAGES) return (ENOTTY); /* * Currently only F_SEAL_SEAL may be set when creating or opening shmfd. * If the decision is made later to allow additional seals, care must be * taken below to ensure that the seals are properly set if the shmfd * already existed -- this currently assumes that only F_SEAL_SEAL can * be set and doesn't take further precautions to ensure the validity of * the seals being added with respect to current mappings. */ if ((initial_seals & ~F_SEAL_SEAL) != 0) return (EINVAL); pdp = td->td_proc->p_pd; cmode = (mode & ~pdp->pd_cmask) & ACCESSPERMS; /* * shm_open(2) created shm should always have O_CLOEXEC set, as mandated * by POSIX. We allow it to be unset here so that an in-kernel * interface may be written as a thin layer around shm, optionally not * setting CLOEXEC. For shm_open(2), O_CLOEXEC is set unconditionally * in sys_shm_open() to keep this implementation compliant. */ error = falloc_caps(td, &fp, &fd, flags & O_CLOEXEC, fcaps); if (error) return (error); /* A SHM_ANON path pointer creates an anonymous object. */ if (userpath == SHM_ANON) { /* A read-only anonymous object is pointless. */ if ((flags & O_ACCMODE) == O_RDONLY) { fdclose(td, fp, fd); fdrop(fp, td); return (EINVAL); } shmfd = shm_alloc(td->td_ucred, cmode, largepage); shmfd->shm_seals = initial_seals; shmfd->shm_flags = shmflags; } else { error = shm_copyin_path(td, userpath, &path); if (error != 0) { fdclose(td, fp, fd); fdrop(fp, td); return (error); } AUDIT_ARG_UPATH1_CANON(path); fnv = fnv_32_str(path, FNV1_32_INIT); sx_xlock(&shm_dict_lock); shmfd = shm_lookup(path, fnv); if (shmfd == NULL) { /* Object does not yet exist, create it if requested. */ if (flags & O_CREAT) { #ifdef MAC error = mac_posixshm_check_create(td->td_ucred, path); if (error == 0) { #endif shmfd = shm_alloc(td->td_ucred, cmode, largepage); shmfd->shm_seals = initial_seals; shmfd->shm_flags = shmflags; shm_insert(path, fnv, shmfd); #ifdef MAC } #endif } else { free(path, M_SHMFD); error = ENOENT; } } else { rl_cookie = rangelock_wlock(&shmfd->shm_rl, 0, OFF_MAX, &shmfd->shm_mtx); /* * kern_shm_open() likely shouldn't ever error out on * trying to set a seal that already exists, unlike * F_ADD_SEALS. This would break terribly as * shm_open(2) actually sets F_SEAL_SEAL to maintain * historical behavior where the underlying file could * not be sealed. */ initial_seals &= ~shmfd->shm_seals; /* * Object already exists, obtain a new * reference if requested and permitted. */ free(path, M_SHMFD); /* * initial_seals can't set additional seals if we've * already been set F_SEAL_SEAL. If F_SEAL_SEAL is set, * then we've already removed that one from * initial_seals. This is currently redundant as we * only allow setting F_SEAL_SEAL at creation time, but * it's cheap to check and decreases the effort required * to allow additional seals. */ if ((shmfd->shm_seals & F_SEAL_SEAL) != 0 && initial_seals != 0) error = EPERM; else if ((flags & (O_CREAT | O_EXCL)) == (O_CREAT | O_EXCL)) error = EEXIST; else if (shmflags != 0 && shmflags != shmfd->shm_flags) error = EINVAL; else { #ifdef MAC error = mac_posixshm_check_open(td->td_ucred, shmfd, FFLAGS(flags & O_ACCMODE)); if (error == 0) #endif error = shm_access(shmfd, td->td_ucred, FFLAGS(flags & O_ACCMODE)); } /* * Truncate the file back to zero length if * O_TRUNC was specified and the object was * opened with read/write. */ if (error == 0 && (flags & (O_ACCMODE | O_TRUNC)) == (O_RDWR | O_TRUNC)) { VM_OBJECT_WLOCK(shmfd->shm_object); #ifdef MAC error = mac_posixshm_check_truncate( td->td_ucred, fp->f_cred, shmfd); if (error == 0) #endif error = shm_dotruncate_locked(shmfd, 0, rl_cookie); VM_OBJECT_WUNLOCK(shmfd->shm_object); } if (error == 0) { /* * Currently we only allow F_SEAL_SEAL to be * set initially. As noted above, this would * need to be reworked should that change. */ shmfd->shm_seals |= initial_seals; shm_hold(shmfd); } rangelock_unlock(&shmfd->shm_rl, rl_cookie, &shmfd->shm_mtx); } sx_xunlock(&shm_dict_lock); if (error) { fdclose(td, fp, fd); fdrop(fp, td); return (error); } } finit(fp, FFLAGS(flags & O_ACCMODE), DTYPE_SHM, shmfd, &shm_ops); td->td_retval[0] = fd; fdrop(fp, td); return (0); } /* System calls. */ #ifdef COMPAT_FREEBSD12 int freebsd12_shm_open(struct thread *td, struct freebsd12_shm_open_args *uap) { return (kern_shm_open(td, uap->path, uap->flags | O_CLOEXEC, uap->mode, NULL)); } #endif int sys_shm_unlink(struct thread *td, struct shm_unlink_args *uap) { char *path; Fnv32_t fnv; int error; error = shm_copyin_path(td, uap->path, &path); if (error != 0) return (error); AUDIT_ARG_UPATH1_CANON(path); fnv = fnv_32_str(path, FNV1_32_INIT); sx_xlock(&shm_dict_lock); error = shm_remove(path, fnv, td->td_ucred); sx_xunlock(&shm_dict_lock); free(path, M_SHMFD); return (error); } int sys_shm_rename(struct thread *td, struct shm_rename_args *uap) { char *path_from = NULL, *path_to = NULL; Fnv32_t fnv_from, fnv_to; struct shmfd *fd_from; struct shmfd *fd_to; int error; int flags; flags = uap->flags; AUDIT_ARG_FFLAGS(flags); /* * Make sure the user passed only valid flags. * If you add a new flag, please add a new term here. */ if ((flags & ~( SHM_RENAME_NOREPLACE | SHM_RENAME_EXCHANGE )) != 0) { error = EINVAL; goto out; } /* * EXCHANGE and NOREPLACE don't quite make sense together. Let's * force the user to choose one or the other. */ if ((flags & SHM_RENAME_NOREPLACE) != 0 && (flags & SHM_RENAME_EXCHANGE) != 0) { error = EINVAL; goto out; } /* Renaming to or from anonymous makes no sense */ if (uap->path_from == SHM_ANON || uap->path_to == SHM_ANON) { error = EINVAL; goto out; } error = shm_copyin_path(td, uap->path_from, &path_from); if (error != 0) goto out; error = shm_copyin_path(td, uap->path_to, &path_to); if (error != 0) goto out; AUDIT_ARG_UPATH1_CANON(path_from); AUDIT_ARG_UPATH2_CANON(path_to); /* Rename with from/to equal is a no-op */ if (strcmp(path_from, path_to) == 0) goto out; fnv_from = fnv_32_str(path_from, FNV1_32_INIT); fnv_to = fnv_32_str(path_to, FNV1_32_INIT); sx_xlock(&shm_dict_lock); fd_from = shm_lookup(path_from, fnv_from); if (fd_from == NULL) { error = ENOENT; goto out_locked; } fd_to = shm_lookup(path_to, fnv_to); if ((flags & SHM_RENAME_NOREPLACE) != 0 && fd_to != NULL) { error = EEXIST; goto out_locked; } /* * Unconditionally prevents shm_remove from invalidating the 'from' * shm's state. */ shm_hold(fd_from); error = shm_remove(path_from, fnv_from, td->td_ucred); /* * One of my assumptions failed if ENOENT (e.g. locking didn't * protect us) */ KASSERT(error != ENOENT, ("Our shm disappeared during shm_rename: %s", path_from)); if (error != 0) { shm_drop(fd_from); goto out_locked; } /* * If we are exchanging, we need to ensure the shm_remove below * doesn't invalidate the dest shm's state. */ if ((flags & SHM_RENAME_EXCHANGE) != 0 && fd_to != NULL) shm_hold(fd_to); /* * NOTE: if path_to is not already in the hash, c'est la vie; * it simply means we have nothing already at path_to to unlink. * That is the ENOENT case. * * If we somehow don't have access to unlink this guy, but * did for the shm at path_from, then relink the shm to path_from * and abort with EACCES. * * All other errors: that is weird; let's relink and abort the * operation. */ error = shm_remove(path_to, fnv_to, td->td_ucred); if (error != 0 && error != ENOENT) { shm_insert(path_from, fnv_from, fd_from); shm_drop(fd_from); /* Don't free path_from now, since the hash references it */ path_from = NULL; goto out_locked; } error = 0; shm_insert(path_to, fnv_to, fd_from); /* Don't free path_to now, since the hash references it */ path_to = NULL; /* We kept a ref when we removed, and incremented again in insert */ shm_drop(fd_from); KASSERT(fd_from->shm_refs > 0, ("Expected >0 refs; got: %d\n", fd_from->shm_refs)); if ((flags & SHM_RENAME_EXCHANGE) != 0 && fd_to != NULL) { shm_insert(path_from, fnv_from, fd_to); path_from = NULL; shm_drop(fd_to); KASSERT(fd_to->shm_refs > 0, ("Expected >0 refs; got: %d\n", fd_to->shm_refs)); } out_locked: sx_xunlock(&shm_dict_lock); out: free(path_from, M_SHMFD); free(path_to, M_SHMFD); return (error); } static int shm_mmap_large(struct shmfd *shmfd, vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot, vm_prot_t max_prot, int flags, vm_ooffset_t foff, struct thread *td) { struct vmspace *vms; vm_map_entry_t next_entry, prev_entry; vm_offset_t align, mask, maxaddr; int docow, error, rv, try; bool curmap; if (shmfd->shm_lp_psind == 0) return (EINVAL); /* MAP_PRIVATE is disabled */ if ((flags & ~(MAP_SHARED | MAP_FIXED | MAP_EXCL | - MAP_NOCORE | -#ifdef MAP_32BIT - MAP_32BIT | -#endif - MAP_ALIGNMENT_MASK)) != 0) + MAP_NOCORE | MAP_32BIT | MAP_ALIGNMENT_MASK)) != 0) return (EINVAL); vms = td->td_proc->p_vmspace; curmap = map == &vms->vm_map; if (curmap) { error = kern_mmap_racct_check(td, map, size); if (error != 0) return (error); } docow = shmfd->shm_lp_psind << MAP_SPLIT_BOUNDARY_SHIFT; docow |= MAP_INHERIT_SHARE; if ((flags & MAP_NOCORE) != 0) docow |= MAP_DISABLE_COREDUMP; mask = pagesizes[shmfd->shm_lp_psind] - 1; if ((foff & mask) != 0) return (EINVAL); maxaddr = vm_map_max(map); -#ifdef MAP_32BIT if ((flags & MAP_32BIT) != 0 && maxaddr > MAP_32BIT_MAX_ADDR) maxaddr = MAP_32BIT_MAX_ADDR; -#endif if (size == 0 || (size & mask) != 0 || (*addr != 0 && ((*addr & mask) != 0 || *addr + size < *addr || *addr + size > maxaddr))) return (EINVAL); align = flags & MAP_ALIGNMENT_MASK; if (align == 0) { align = pagesizes[shmfd->shm_lp_psind]; } else if (align == MAP_ALIGNED_SUPER) { if (shmfd->shm_lp_psind != 1) return (EINVAL); align = pagesizes[1]; } else { align >>= MAP_ALIGNMENT_SHIFT; align = 1ULL << align; /* Also handles overflow. */ if (align < pagesizes[shmfd->shm_lp_psind]) return (EINVAL); } vm_map_lock(map); if ((flags & MAP_FIXED) == 0) { try = 1; if (curmap && (*addr == 0 || (*addr >= round_page((vm_offset_t)vms->vm_taddr) && *addr < round_page((vm_offset_t)vms->vm_daddr + lim_max(td, RLIMIT_DATA))))) { *addr = roundup2((vm_offset_t)vms->vm_daddr + lim_max(td, RLIMIT_DATA), pagesizes[shmfd->shm_lp_psind]); } again: rv = vm_map_find_aligned(map, addr, size, maxaddr, align); if (rv != KERN_SUCCESS) { if (try == 1) { try = 2; *addr = vm_map_min(map); if ((*addr & mask) != 0) *addr = (*addr + mask) & mask; goto again; } goto fail1; } } else if ((flags & MAP_EXCL) == 0) { rv = vm_map_delete(map, *addr, *addr + size); if (rv != KERN_SUCCESS) goto fail1; } else { error = ENOSPC; if (vm_map_lookup_entry(map, *addr, &prev_entry)) goto fail; next_entry = vm_map_entry_succ(prev_entry); if (next_entry->start < *addr + size) goto fail; } rv = vm_map_insert(map, shmfd->shm_object, foff, *addr, *addr + size, prot, max_prot, docow); fail1: error = vm_mmap_to_errno(rv); fail: vm_map_unlock(map); return (error); } static int shm_mmap(struct file *fp, vm_map_t map, vm_offset_t *addr, vm_size_t objsize, vm_prot_t prot, vm_prot_t cap_maxprot, int flags, vm_ooffset_t foff, struct thread *td) { struct shmfd *shmfd; vm_prot_t maxprot; int error; bool writecnt; void *rl_cookie; shmfd = fp->f_data; maxprot = VM_PROT_NONE; rl_cookie = rangelock_rlock(&shmfd->shm_rl, 0, objsize, &shmfd->shm_mtx); /* FREAD should always be set. */ if ((fp->f_flag & FREAD) != 0) maxprot |= VM_PROT_EXECUTE | VM_PROT_READ; /* * If FWRITE's set, we can allow VM_PROT_WRITE unless it's a shared * mapping with a write seal applied. Private mappings are always * writeable. */ if ((flags & MAP_SHARED) == 0) { cap_maxprot |= VM_PROT_WRITE; maxprot |= VM_PROT_WRITE; writecnt = false; } else { if ((fp->f_flag & FWRITE) != 0 && (shmfd->shm_seals & F_SEAL_WRITE) == 0) maxprot |= VM_PROT_WRITE; /* * Any mappings from a writable descriptor may be upgraded to * VM_PROT_WRITE with mprotect(2), unless a write-seal was * applied between the open and subsequent mmap(2). We want to * reject application of a write seal as long as any such * mapping exists so that the seal cannot be trivially bypassed. */ writecnt = (maxprot & VM_PROT_WRITE) != 0; if (!writecnt && (prot & VM_PROT_WRITE) != 0) { error = EACCES; goto out; } } maxprot &= cap_maxprot; /* See comment in vn_mmap(). */ if ( #ifdef _LP64 objsize > OFF_MAX || #endif foff > OFF_MAX - objsize) { error = EINVAL; goto out; } #ifdef MAC error = mac_posixshm_check_mmap(td->td_ucred, shmfd, prot, flags); if (error != 0) goto out; #endif mtx_lock(&shm_timestamp_lock); vfs_timestamp(&shmfd->shm_atime); mtx_unlock(&shm_timestamp_lock); vm_object_reference(shmfd->shm_object); if (shm_largepage(shmfd)) { writecnt = false; error = shm_mmap_large(shmfd, map, addr, objsize, prot, maxprot, flags, foff, td); } else { if (writecnt) { vm_pager_update_writecount(shmfd->shm_object, 0, objsize); } error = vm_mmap_object(map, addr, objsize, prot, maxprot, flags, shmfd->shm_object, foff, writecnt, td); } if (error != 0) { if (writecnt) vm_pager_release_writecount(shmfd->shm_object, 0, objsize); vm_object_deallocate(shmfd->shm_object); } out: rangelock_unlock(&shmfd->shm_rl, rl_cookie, &shmfd->shm_mtx); return (error); } static int shm_chmod(struct file *fp, mode_t mode, struct ucred *active_cred, struct thread *td) { struct shmfd *shmfd; int error; error = 0; shmfd = fp->f_data; mtx_lock(&shm_timestamp_lock); /* * SUSv4 says that x bits of permission need not be affected. * Be consistent with our shm_open there. */ #ifdef MAC error = mac_posixshm_check_setmode(active_cred, shmfd, mode); if (error != 0) goto out; #endif error = vaccess(VREG, shmfd->shm_mode, shmfd->shm_uid, shmfd->shm_gid, VADMIN, active_cred); if (error != 0) goto out; shmfd->shm_mode = mode & ACCESSPERMS; out: mtx_unlock(&shm_timestamp_lock); return (error); } static int shm_chown(struct file *fp, uid_t uid, gid_t gid, struct ucred *active_cred, struct thread *td) { struct shmfd *shmfd; int error; error = 0; shmfd = fp->f_data; mtx_lock(&shm_timestamp_lock); #ifdef MAC error = mac_posixshm_check_setowner(active_cred, shmfd, uid, gid); if (error != 0) goto out; #endif if (uid == (uid_t)-1) uid = shmfd->shm_uid; if (gid == (gid_t)-1) gid = shmfd->shm_gid; if (((uid != shmfd->shm_uid && uid != active_cred->cr_uid) || (gid != shmfd->shm_gid && !groupmember(gid, active_cred))) && (error = priv_check_cred(active_cred, PRIV_VFS_CHOWN))) goto out; shmfd->shm_uid = uid; shmfd->shm_gid = gid; out: mtx_unlock(&shm_timestamp_lock); return (error); } /* * Helper routines to allow the backing object of a shared memory file * descriptor to be mapped in the kernel. */ int shm_map(struct file *fp, size_t size, off_t offset, void **memp) { struct shmfd *shmfd; vm_offset_t kva, ofs; vm_object_t obj; int rv; if (fp->f_type != DTYPE_SHM) return (EINVAL); shmfd = fp->f_data; obj = shmfd->shm_object; VM_OBJECT_WLOCK(obj); /* * XXXRW: This validation is probably insufficient, and subject to * sign errors. It should be fixed. */ if (offset >= shmfd->shm_size || offset + size > round_page(shmfd->shm_size)) { VM_OBJECT_WUNLOCK(obj); return (EINVAL); } shmfd->shm_kmappings++; vm_object_reference_locked(obj); VM_OBJECT_WUNLOCK(obj); /* Map the object into the kernel_map and wire it. */ kva = vm_map_min(kernel_map); ofs = offset & PAGE_MASK; offset = trunc_page(offset); size = round_page(size + ofs); rv = vm_map_find(kernel_map, obj, offset, &kva, size, 0, VMFS_OPTIMAL_SPACE, VM_PROT_READ | VM_PROT_WRITE, VM_PROT_READ | VM_PROT_WRITE, 0); if (rv == KERN_SUCCESS) { rv = vm_map_wire(kernel_map, kva, kva + size, VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES); if (rv == KERN_SUCCESS) { *memp = (void *)(kva + ofs); return (0); } vm_map_remove(kernel_map, kva, kva + size); } else vm_object_deallocate(obj); /* On failure, drop our mapping reference. */ VM_OBJECT_WLOCK(obj); shmfd->shm_kmappings--; VM_OBJECT_WUNLOCK(obj); return (vm_mmap_to_errno(rv)); } /* * We require the caller to unmap the entire entry. This allows us to * safely decrement shm_kmappings when a mapping is removed. */ int shm_unmap(struct file *fp, void *mem, size_t size) { struct shmfd *shmfd; vm_map_entry_t entry; vm_offset_t kva, ofs; vm_object_t obj; vm_pindex_t pindex; vm_prot_t prot; boolean_t wired; vm_map_t map; int rv; if (fp->f_type != DTYPE_SHM) return (EINVAL); shmfd = fp->f_data; kva = (vm_offset_t)mem; ofs = kva & PAGE_MASK; kva = trunc_page(kva); size = round_page(size + ofs); map = kernel_map; rv = vm_map_lookup(&map, kva, VM_PROT_READ | VM_PROT_WRITE, &entry, &obj, &pindex, &prot, &wired); if (rv != KERN_SUCCESS) return (EINVAL); if (entry->start != kva || entry->end != kva + size) { vm_map_lookup_done(map, entry); return (EINVAL); } vm_map_lookup_done(map, entry); if (obj != shmfd->shm_object) return (EINVAL); vm_map_remove(map, kva, kva + size); VM_OBJECT_WLOCK(obj); KASSERT(shmfd->shm_kmappings > 0, ("shm_unmap: object not mapped")); shmfd->shm_kmappings--; VM_OBJECT_WUNLOCK(obj); return (0); } static int shm_fill_kinfo_locked(struct shmfd *shmfd, struct kinfo_file *kif, bool list) { const char *path, *pr_path; size_t pr_pathlen; bool visible; sx_assert(&shm_dict_lock, SA_LOCKED); kif->kf_type = KF_TYPE_SHM; kif->kf_un.kf_file.kf_file_mode = S_IFREG | shmfd->shm_mode; kif->kf_un.kf_file.kf_file_size = shmfd->shm_size; if (shmfd->shm_path != NULL) { if (shmfd->shm_path != NULL) { path = shmfd->shm_path; pr_path = curthread->td_ucred->cr_prison->pr_path; if (strcmp(pr_path, "/") != 0) { /* Return the jail-rooted pathname. */ pr_pathlen = strlen(pr_path); visible = strncmp(path, pr_path, pr_pathlen) == 0 && path[pr_pathlen] == '/'; if (list && !visible) return (EPERM); if (visible) path += pr_pathlen; } strlcpy(kif->kf_path, path, sizeof(kif->kf_path)); } } return (0); } static int shm_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp __unused) { int res; sx_slock(&shm_dict_lock); res = shm_fill_kinfo_locked(fp->f_data, kif, false); sx_sunlock(&shm_dict_lock); return (res); } static int shm_add_seals(struct file *fp, int seals) { struct shmfd *shmfd; void *rl_cookie; vm_ooffset_t writemappings; int error, nseals; error = 0; shmfd = fp->f_data; rl_cookie = rangelock_wlock(&shmfd->shm_rl, 0, OFF_MAX, &shmfd->shm_mtx); /* Even already-set seals should result in EPERM. */ if ((shmfd->shm_seals & F_SEAL_SEAL) != 0) { error = EPERM; goto out; } nseals = seals & ~shmfd->shm_seals; if ((nseals & F_SEAL_WRITE) != 0) { if (shm_largepage(shmfd)) { error = ENOTSUP; goto out; } /* * The rangelock above prevents writable mappings from being * added after we've started applying seals. The RLOCK here * is to avoid torn reads on ILP32 arches as unmapping/reducing * writemappings will be done without a rangelock. */ VM_OBJECT_RLOCK(shmfd->shm_object); writemappings = shmfd->shm_object->un_pager.swp.writemappings; VM_OBJECT_RUNLOCK(shmfd->shm_object); /* kmappings are also writable */ if (writemappings > 0) { error = EBUSY; goto out; } } shmfd->shm_seals |= nseals; out: rangelock_unlock(&shmfd->shm_rl, rl_cookie, &shmfd->shm_mtx); return (error); } static int shm_get_seals(struct file *fp, int *seals) { struct shmfd *shmfd; shmfd = fp->f_data; *seals = shmfd->shm_seals; return (0); } static int shm_deallocate(struct shmfd *shmfd, off_t *offset, off_t *length, int flags) { vm_object_t object; vm_pindex_t pistart, pi, piend; vm_ooffset_t off, len; int startofs, endofs, end; int error; off = *offset; len = *length; KASSERT(off + len <= (vm_ooffset_t)OFF_MAX, ("off + len overflows")); if (off + len > shmfd->shm_size) len = shmfd->shm_size - off; object = shmfd->shm_object; startofs = off & PAGE_MASK; endofs = (off + len) & PAGE_MASK; pistart = OFF_TO_IDX(off); piend = OFF_TO_IDX(off + len); pi = OFF_TO_IDX(off + PAGE_MASK); error = 0; /* Handle the case when offset is on or beyond shm size. */ if ((off_t)len <= 0) { *length = 0; return (0); } VM_OBJECT_WLOCK(object); if (startofs != 0) { end = pistart != piend ? PAGE_SIZE : endofs; error = shm_partial_page_invalidate(object, pistart, startofs, end); if (error) goto out; off += end - startofs; len -= end - startofs; } if (pi < piend) { vm_object_page_remove(object, pi, piend, 0); off += IDX_TO_OFF(piend - pi); len -= IDX_TO_OFF(piend - pi); } if (endofs != 0 && pistart != piend) { error = shm_partial_page_invalidate(object, piend, 0, endofs); if (error) goto out; off += endofs; len -= endofs; } out: VM_OBJECT_WUNLOCK(shmfd->shm_object); *offset = off; *length = len; return (error); } static int shm_fspacectl(struct file *fp, int cmd, off_t *offset, off_t *length, int flags, struct ucred *active_cred, struct thread *td) { void *rl_cookie; struct shmfd *shmfd; off_t off, len; int error; KASSERT(cmd == SPACECTL_DEALLOC, ("shm_fspacectl: Invalid cmd")); KASSERT((flags & ~SPACECTL_F_SUPPORTED) == 0, ("shm_fspacectl: non-zero flags")); KASSERT(*offset >= 0 && *length > 0 && *length <= OFF_MAX - *offset, ("shm_fspacectl: offset/length overflow or underflow")); error = EINVAL; shmfd = fp->f_data; off = *offset; len = *length; rl_cookie = rangelock_wlock(&shmfd->shm_rl, off, off + len, &shmfd->shm_mtx); switch (cmd) { case SPACECTL_DEALLOC: if ((shmfd->shm_seals & F_SEAL_WRITE) != 0) { error = EPERM; break; } error = shm_deallocate(shmfd, &off, &len, flags); *offset = off; *length = len; break; default: __assert_unreachable(); } rangelock_unlock(&shmfd->shm_rl, rl_cookie, &shmfd->shm_mtx); return (error); } static int shm_fallocate(struct file *fp, off_t offset, off_t len, struct thread *td) { void *rl_cookie; struct shmfd *shmfd; size_t size; int error; /* This assumes that the caller already checked for overflow. */ error = 0; shmfd = fp->f_data; size = offset + len; /* * Just grab the rangelock for the range that we may be attempting to * grow, rather than blocking read/write for regions we won't be * touching while this (potential) resize is in progress. Other * attempts to resize the shmfd will have to take a write lock from 0 to * OFF_MAX, so this being potentially beyond the current usable range of * the shmfd is not necessarily a concern. If other mechanisms are * added to grow a shmfd, this may need to be re-evaluated. */ rl_cookie = rangelock_wlock(&shmfd->shm_rl, offset, size, &shmfd->shm_mtx); if (size > shmfd->shm_size) error = shm_dotruncate_cookie(shmfd, size, rl_cookie); rangelock_unlock(&shmfd->shm_rl, rl_cookie, &shmfd->shm_mtx); /* Translate to posix_fallocate(2) return value as needed. */ if (error == ENOMEM) error = ENOSPC; return (error); } static int sysctl_posix_shm_list(SYSCTL_HANDLER_ARGS) { struct shm_mapping *shmm; struct sbuf sb; struct kinfo_file kif; u_long i; int error, error2; sbuf_new_for_sysctl(&sb, NULL, sizeof(struct kinfo_file) * 5, req); sbuf_clear_flags(&sb, SBUF_INCLUDENUL); error = 0; sx_slock(&shm_dict_lock); for (i = 0; i < shm_hash + 1; i++) { LIST_FOREACH(shmm, &shm_dictionary[i], sm_link) { error = shm_fill_kinfo_locked(shmm->sm_shmfd, &kif, true); if (error == EPERM) { error = 0; continue; } if (error != 0) break; pack_kinfo(&kif); error = sbuf_bcat(&sb, &kif, kif.kf_structsize) == 0 ? 0 : ENOMEM; if (error != 0) break; } } sx_sunlock(&shm_dict_lock); error2 = sbuf_finish(&sb); sbuf_delete(&sb); return (error != 0 ? error : error2); } SYSCTL_PROC(_kern_ipc, OID_AUTO, posix_shm_list, CTLFLAG_RD | CTLFLAG_PRISON | CTLFLAG_MPSAFE | CTLTYPE_OPAQUE, NULL, 0, sysctl_posix_shm_list, "", "POSIX SHM list"); int kern_shm_open(struct thread *td, const char *path, int flags, mode_t mode, struct filecaps *caps) { return (kern_shm_open2(td, path, flags, mode, 0, caps, NULL)); } /* * This version of the shm_open() interface leaves CLOEXEC behavior up to the * caller, and libc will enforce it for the traditional shm_open() call. This * allows other consumers, like memfd_create(), to opt-in for CLOEXEC. This * interface also includes a 'name' argument that is currently unused, but could * potentially be exported later via some interface for debugging purposes. * From the kernel's perspective, it is optional. Individual consumers like * memfd_create() may require it in order to be compatible with other systems * implementing the same function. */ int sys_shm_open2(struct thread *td, struct shm_open2_args *uap) { return (kern_shm_open2(td, uap->path, uap->flags, uap->mode, uap->shmflags, NULL, uap->name)); } diff --git a/sys/sys/mman.h b/sys/sys/mman.h index 0413249dee2c..29b00107792f 100644 --- a/sys/sys/mman.h +++ b/sys/sys/mman.h @@ -1,361 +1,359 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 1982, 1986, 1993 * The Regents of the University of California. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)mman.h 8.2 (Berkeley) 1/9/95 * $FreeBSD$ */ #ifndef _SYS_MMAN_H_ #define _SYS_MMAN_H_ #include #include #if __BSD_VISIBLE /* * Inheritance for minherit() */ #define INHERIT_SHARE 0 #define INHERIT_COPY 1 #define INHERIT_NONE 2 #define INHERIT_ZERO 3 #endif /* * Protections are chosen from these bits, or-ed together */ #define PROT_NONE 0x00 /* no permissions */ #define PROT_READ 0x01 /* pages can be read */ #define PROT_WRITE 0x02 /* pages can be written */ #define PROT_EXEC 0x04 /* pages can be executed */ #if __BSD_VISIBLE #define _PROT_ALL (PROT_READ | PROT_WRITE | PROT_EXEC) #define PROT_EXTRACT(prot) ((prot) & _PROT_ALL) #define _PROT_MAX_SHIFT 16 #define PROT_MAX(prot) ((prot) << _PROT_MAX_SHIFT) #define PROT_MAX_EXTRACT(prot) (((prot) >> _PROT_MAX_SHIFT) & _PROT_ALL) #endif /* * Flags contain sharing type and options. * Sharing types; choose one. */ #define MAP_SHARED 0x0001 /* share changes */ #define MAP_PRIVATE 0x0002 /* changes are private */ #if __BSD_VISIBLE #define MAP_COPY MAP_PRIVATE /* Obsolete */ #endif /* * Other flags */ #define MAP_FIXED 0x0010 /* map addr must be exactly as requested */ #if __BSD_VISIBLE #define MAP_RESERVED0020 0x0020 /* previously unimplemented MAP_RENAME */ #define MAP_RESERVED0040 0x0040 /* previously unimplemented MAP_NORESERVE */ #define MAP_RESERVED0080 0x0080 /* previously misimplemented MAP_INHERIT */ #define MAP_RESERVED0100 0x0100 /* previously unimplemented MAP_NOEXTEND */ #define MAP_HASSEMAPHORE 0x0200 /* region may contain semaphores */ #define MAP_STACK 0x0400 /* region grows down, like a stack */ #define MAP_NOSYNC 0x0800 /* page to but do not sync underlying file */ /* * Mapping type */ #define MAP_FILE 0x0000 /* map from file (default) */ #define MAP_ANON 0x1000 /* allocated from memory, swap space */ #ifndef _KERNEL #define MAP_ANONYMOUS MAP_ANON /* For compatibility. */ #endif /* !_KERNEL */ /* * Extended flags */ #define MAP_GUARD 0x00002000 /* reserve but don't map address range */ #define MAP_EXCL 0x00004000 /* for MAP_FIXED, fail if address is used */ #define MAP_NOCORE 0x00020000 /* dont include these pages in a coredump */ #define MAP_PREFAULT_READ 0x00040000 /* prefault mapping for reading */ -#ifdef __LP64__ #define MAP_32BIT 0x00080000 /* map in the low 2GB of address space */ -#endif /* * Request specific alignment (n == log2 of the desired alignment). * * MAP_ALIGNED_SUPER requests optimal superpage alignment, but does * not enforce a specific alignment. */ #define MAP_ALIGNED(n) ((n) << MAP_ALIGNMENT_SHIFT) #define MAP_ALIGNMENT_SHIFT 24 #define MAP_ALIGNMENT_MASK MAP_ALIGNED(0xff) #define MAP_ALIGNED_SUPER MAP_ALIGNED(1) /* align on a superpage */ /* * Flags provided to shm_rename */ /* Don't overwrite dest, if it exists */ #define SHM_RENAME_NOREPLACE (1 << 0) /* Atomically swap src and dest */ #define SHM_RENAME_EXCHANGE (1 << 1) #endif /* __BSD_VISIBLE */ #if __POSIX_VISIBLE >= 199309 /* * Process memory locking */ #define MCL_CURRENT 0x0001 /* Lock only current memory */ #define MCL_FUTURE 0x0002 /* Lock all future memory as well */ #endif /* * Error return from mmap() */ #define MAP_FAILED ((void *)-1) /* * msync() flags */ #define MS_SYNC 0x0000 /* msync synchronously */ #define MS_ASYNC 0x0001 /* return immediately */ #define MS_INVALIDATE 0x0002 /* invalidate all cached data */ /* * Advice to madvise */ #define _MADV_NORMAL 0 /* no further special treatment */ #define _MADV_RANDOM 1 /* expect random page references */ #define _MADV_SEQUENTIAL 2 /* expect sequential page references */ #define _MADV_WILLNEED 3 /* will need these pages */ #define _MADV_DONTNEED 4 /* dont need these pages */ #if __BSD_VISIBLE #define MADV_NORMAL _MADV_NORMAL #define MADV_RANDOM _MADV_RANDOM #define MADV_SEQUENTIAL _MADV_SEQUENTIAL #define MADV_WILLNEED _MADV_WILLNEED #define MADV_DONTNEED _MADV_DONTNEED #define MADV_FREE 5 /* dont need these pages, and junk contents */ #define MADV_NOSYNC 6 /* try to avoid flushes to physical media */ #define MADV_AUTOSYNC 7 /* revert to default flushing strategy */ #define MADV_NOCORE 8 /* do not include these pages in a core file */ #define MADV_CORE 9 /* revert to including pages in a core file */ #define MADV_PROTECT 10 /* protect process from pageout kill */ /* * Return bits from mincore */ #define MINCORE_INCORE 0x1 /* Page is incore */ #define MINCORE_REFERENCED 0x2 /* Page has been referenced by us */ #define MINCORE_MODIFIED 0x4 /* Page has been modified by us */ #define MINCORE_REFERENCED_OTHER 0x8 /* Page has been referenced */ #define MINCORE_MODIFIED_OTHER 0x10 /* Page has been modified */ #define MINCORE_SUPER 0x60 /* Page is a "super" page */ #define MINCORE_PSIND(i) (((i) << 5) & MINCORE_SUPER) /* Page size */ /* * Anonymous object constant for shm_open(). */ #define SHM_ANON ((char *)1) /* * shmflags for shm_open2() */ #define SHM_ALLOW_SEALING 0x00000001 #define SHM_GROW_ON_WRITE 0x00000002 #define SHM_LARGEPAGE 0x00000004 #define SHM_LARGEPAGE_ALLOC_DEFAULT 0 #define SHM_LARGEPAGE_ALLOC_NOWAIT 1 #define SHM_LARGEPAGE_ALLOC_HARD 2 struct shm_largepage_conf { int psind; int alloc_policy; int pad[10]; }; /* * Flags for memfd_create(). */ #define MFD_CLOEXEC 0x00000001 #define MFD_ALLOW_SEALING 0x00000002 #define MFD_HUGETLB 0x00000004 #define MFD_HUGE_MASK 0xFC000000 #define MFD_HUGE_SHIFT 26 #define MFD_HUGE_64KB (16 << MFD_HUGE_SHIFT) #define MFD_HUGE_512KB (19 << MFD_HUGE_SHIFT) #define MFD_HUGE_1MB (20 << MFD_HUGE_SHIFT) #define MFD_HUGE_2MB (21 << MFD_HUGE_SHIFT) #define MFD_HUGE_8MB (23 << MFD_HUGE_SHIFT) #define MFD_HUGE_16MB (24 << MFD_HUGE_SHIFT) #define MFD_HUGE_32MB (25 << MFD_HUGE_SHIFT) #define MFD_HUGE_256MB (28 << MFD_HUGE_SHIFT) #define MFD_HUGE_512MB (29 << MFD_HUGE_SHIFT) #define MFD_HUGE_1GB (30 << MFD_HUGE_SHIFT) #define MFD_HUGE_2GB (31 << MFD_HUGE_SHIFT) #define MFD_HUGE_16GB (34 << MFD_HUGE_SHIFT) #endif /* __BSD_VISIBLE */ /* * XXX missing POSIX_TYPED_MEM_* macros and * posix_typed_mem_info structure. */ #if __POSIX_VISIBLE >= 200112 #define POSIX_MADV_NORMAL _MADV_NORMAL #define POSIX_MADV_RANDOM _MADV_RANDOM #define POSIX_MADV_SEQUENTIAL _MADV_SEQUENTIAL #define POSIX_MADV_WILLNEED _MADV_WILLNEED #define POSIX_MADV_DONTNEED _MADV_DONTNEED #endif #ifndef _MODE_T_DECLARED typedef __mode_t mode_t; #define _MODE_T_DECLARED #endif #ifndef _OFF_T_DECLARED typedef __off_t off_t; #define _OFF_T_DECLARED #endif #ifndef _SIZE_T_DECLARED typedef __size_t size_t; #define _SIZE_T_DECLARED #endif #if defined(_KERNEL) || defined(_WANT_FILE) #include #include #include #include #include struct file; struct shmfd { vm_ooffset_t shm_size; vm_object_t shm_object; vm_pindex_t shm_pages; /* allocated pages */ int shm_refs; uid_t shm_uid; gid_t shm_gid; mode_t shm_mode; int shm_kmappings; /* * Values maintained solely to make this a better-behaved file * descriptor for fstat() to run on. */ struct timespec shm_atime; struct timespec shm_mtime; struct timespec shm_ctime; struct timespec shm_birthtime; ino_t shm_ino; struct label *shm_label; /* MAC label */ const char *shm_path; struct rangelock shm_rl; struct mtx shm_mtx; int shm_flags; int shm_seals; /* largepage config */ int shm_lp_psind; int shm_lp_alloc_policy; }; #endif #ifdef _KERNEL struct prison; int shm_map(struct file *fp, size_t size, off_t offset, void **memp); int shm_unmap(struct file *fp, void *mem, size_t size); int shm_access(struct shmfd *shmfd, struct ucred *ucred, int flags); struct shmfd *shm_alloc(struct ucred *ucred, mode_t mode, bool largepage); struct shmfd *shm_hold(struct shmfd *shmfd); void shm_drop(struct shmfd *shmfd); int shm_dotruncate(struct shmfd *shmfd, off_t length); bool shm_largepage(struct shmfd *shmfd); void shm_remove_prison(struct prison *pr); extern struct fileops shm_ops; #define MAP_32BIT_MAX_ADDR ((vm_offset_t)1 << 31) #else /* !_KERNEL */ __BEGIN_DECLS /* * XXX not yet implemented: posix_mem_offset(), posix_typed_mem_get_info(), * posix_typed_mem_open(). */ #if __BSD_VISIBLE int getpagesizes(size_t *, int); int madvise(void *, size_t, int); int mincore(const void *, size_t, char *); int minherit(void *, size_t, int); #endif int mlock(const void *, size_t); #ifndef _MMAP_DECLARED #define _MMAP_DECLARED void * mmap(void *, size_t, int, int, int, off_t); #endif int mprotect(void *, size_t, int); int msync(void *, size_t, int); int munlock(const void *, size_t); int munmap(void *, size_t); #if __POSIX_VISIBLE >= 200112 int posix_madvise(void *, size_t, int); #endif #if __POSIX_VISIBLE >= 199309 int mlockall(int); int munlockall(void); int shm_open(const char *, int, mode_t); int shm_unlink(const char *); #endif #if __BSD_VISIBLE int memfd_create(const char *, unsigned int); int shm_create_largepage(const char *, int, int, int, mode_t); int shm_rename(const char *, const char *, int); #endif __END_DECLS #endif /* !_KERNEL */ #endif /* !_SYS_MMAN_H_ */ diff --git a/sys/vm/vm_mmap.c b/sys/vm/vm_mmap.c index 64287f096332..8f0b985da627 100644 --- a/sys/vm/vm_mmap.c +++ b/sys/vm/vm_mmap.c @@ -1,1685 +1,1672 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 1988 University of Utah. * Copyright (c) 1991, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * the Systems Programming Group of the University of Utah Computer * Science Department. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: Utah $Hdr: vm_mmap.c 1.6 91/10/21$ * * @(#)vm_mmap.c 8.4 (Berkeley) 1/12/94 */ /* * Mapped file (mmap) interface to VM */ #include __FBSDID("$FreeBSD$"); #include "opt_hwpmc_hooks.h" #include "opt_vm.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(__amd64__) || defined(__i386__) /* for i386_read_exec */ #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef HWPMC_HOOKS #include #endif int old_mlock = 0; SYSCTL_INT(_vm, OID_AUTO, old_mlock, CTLFLAG_RWTUN, &old_mlock, 0, "Do not apply RLIMIT_MEMLOCK on mlockall"); static int mincore_mapped = 1; SYSCTL_INT(_vm, OID_AUTO, mincore_mapped, CTLFLAG_RWTUN, &mincore_mapped, 0, "mincore reports mappings, not residency"); static int imply_prot_max = 0; SYSCTL_INT(_vm, OID_AUTO, imply_prot_max, CTLFLAG_RWTUN, &imply_prot_max, 0, "Imply maximum page protections in mmap() when none are specified"); _Static_assert(MAXPAGESIZES <= 4, "MINCORE_SUPER too narrow"); #ifndef _SYS_SYSPROTO_H_ struct sbrk_args { int incr; }; #endif int sys_sbrk(struct thread *td, struct sbrk_args *uap) { /* Not yet implemented */ return (EOPNOTSUPP); } #ifndef _SYS_SYSPROTO_H_ struct sstk_args { int incr; }; #endif int sys_sstk(struct thread *td, struct sstk_args *uap) { /* Not yet implemented */ return (EOPNOTSUPP); } #if defined(COMPAT_43) int ogetpagesize(struct thread *td, struct ogetpagesize_args *uap) { td->td_retval[0] = PAGE_SIZE; return (0); } #endif /* COMPAT_43 */ /* * Memory Map (mmap) system call. Note that the file offset * and address are allowed to be NOT page aligned, though if * the MAP_FIXED flag it set, both must have the same remainder * modulo the PAGE_SIZE (POSIX 1003.1b). If the address is not * page-aligned, the actual mapping starts at trunc_page(addr) * and the return value is adjusted up by the page offset. * * Generally speaking, only character devices which are themselves * memory-based, such as a video framebuffer, can be mmap'd. Otherwise * there would be no cache coherency between a descriptor and a VM mapping * both to the same character device. */ #ifndef _SYS_SYSPROTO_H_ struct mmap_args { void *addr; size_t len; int prot; int flags; int fd; long pad; off_t pos; }; #endif int sys_mmap(struct thread *td, struct mmap_args *uap) { return (kern_mmap(td, &(struct mmap_req){ .mr_hint = (uintptr_t)uap->addr, .mr_len = uap->len, .mr_prot = uap->prot, .mr_flags = uap->flags, .mr_fd = uap->fd, .mr_pos = uap->pos, })); } int kern_mmap_maxprot(struct proc *p, int prot) { if ((p->p_flag2 & P2_PROTMAX_DISABLE) != 0 || (p->p_fctl0 & NT_FREEBSD_FCTL_PROTMAX_DISABLE) != 0) return (_PROT_ALL); if (((p->p_flag2 & P2_PROTMAX_ENABLE) != 0 || imply_prot_max) && prot != PROT_NONE) return (prot); return (_PROT_ALL); } int kern_mmap(struct thread *td, const struct mmap_req *mrp) { struct vmspace *vms; struct file *fp; struct proc *p; off_t pos; vm_offset_t addr, orig_addr; vm_size_t len, pageoff, size; vm_prot_t cap_maxprot; int align, error, fd, flags, max_prot, prot; cap_rights_t rights; mmap_check_fp_fn check_fp_fn; orig_addr = addr = mrp->mr_hint; len = mrp->mr_len; prot = mrp->mr_prot; flags = mrp->mr_flags; fd = mrp->mr_fd; pos = mrp->mr_pos; check_fp_fn = mrp->mr_check_fp_fn; if ((prot & ~(_PROT_ALL | PROT_MAX(_PROT_ALL))) != 0) return (EINVAL); max_prot = PROT_MAX_EXTRACT(prot); prot = PROT_EXTRACT(prot); if (max_prot != 0 && (max_prot & prot) != prot) return (ENOTSUP); p = td->td_proc; /* * Always honor PROT_MAX if set. If not, default to all * permissions unless we're implying maximum permissions. */ if (max_prot == 0) max_prot = kern_mmap_maxprot(p, prot); vms = p->p_vmspace; fp = NULL; AUDIT_ARG_FD(fd); /* * Ignore old flags that used to be defined but did not do anything. */ flags &= ~(MAP_RESERVED0020 | MAP_RESERVED0040); /* * Enforce the constraints. * Mapping of length 0 is only allowed for old binaries. * Anonymous mapping shall specify -1 as filedescriptor and * zero position for new code. Be nice to ancient a.out * binaries and correct pos for anonymous mapping, since old * ld.so sometimes issues anonymous map requests with non-zero * pos. */ if (!SV_CURPROC_FLAG(SV_AOUT)) { if ((len == 0 && p->p_osrel >= P_OSREL_MAP_ANON) || ((flags & MAP_ANON) != 0 && (fd != -1 || pos != 0))) return (EINVAL); } else { if ((flags & MAP_ANON) != 0) pos = 0; } if (flags & MAP_STACK) { if ((fd != -1) || ((prot & (PROT_READ | PROT_WRITE)) != (PROT_READ | PROT_WRITE))) return (EINVAL); flags |= MAP_ANON; pos = 0; } if ((flags & ~(MAP_SHARED | MAP_PRIVATE | MAP_FIXED | MAP_HASSEMAPHORE | MAP_STACK | MAP_NOSYNC | MAP_ANON | MAP_EXCL | MAP_NOCORE | - MAP_PREFAULT_READ | MAP_GUARD | -#ifdef MAP_32BIT - MAP_32BIT | -#endif - MAP_ALIGNMENT_MASK)) != 0) + MAP_PREFAULT_READ | MAP_GUARD | MAP_32BIT | MAP_ALIGNMENT_MASK)) != 0) return (EINVAL); if ((flags & (MAP_EXCL | MAP_FIXED)) == MAP_EXCL) return (EINVAL); if ((flags & (MAP_SHARED | MAP_PRIVATE)) == (MAP_SHARED | MAP_PRIVATE)) return (EINVAL); if (prot != PROT_NONE && (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC)) != 0) return (EINVAL); if ((flags & MAP_GUARD) != 0 && (prot != PROT_NONE || fd != -1 || pos != 0 || (flags & ~(MAP_FIXED | MAP_GUARD | MAP_EXCL | -#ifdef MAP_32BIT - MAP_32BIT | -#endif - MAP_ALIGNMENT_MASK)) != 0)) + MAP_32BIT | MAP_ALIGNMENT_MASK)) != 0)) return (EINVAL); /* * Align the file position to a page boundary, * and save its page offset component. */ pageoff = (pos & PAGE_MASK); pos -= pageoff; /* Compute size from len by rounding (on both ends). */ size = len + pageoff; /* low end... */ size = round_page(size); /* hi end */ /* Check for rounding up to zero. */ if (len > size) return (ENOMEM); /* Ensure alignment is at least a page and fits in a pointer. */ align = flags & MAP_ALIGNMENT_MASK; if (align != 0 && align != MAP_ALIGNED_SUPER && (align >> MAP_ALIGNMENT_SHIFT >= sizeof(void *) * NBBY || align >> MAP_ALIGNMENT_SHIFT < PAGE_SHIFT)) return (EINVAL); /* * Check for illegal addresses. Watch out for address wrap... Note * that VM_*_ADDRESS are not constants due to casts (argh). */ if (flags & MAP_FIXED) { /* * The specified address must have the same remainder * as the file offset taken modulo PAGE_SIZE, so it * should be aligned after adjustment by pageoff. */ addr -= pageoff; if (addr & PAGE_MASK) return (EINVAL); /* Address range must be all in user VM space. */ if (!vm_map_range_valid(&vms->vm_map, addr, addr + size)) return (EINVAL); -#ifdef MAP_32BIT if (flags & MAP_32BIT && addr + size > MAP_32BIT_MAX_ADDR) return (EINVAL); } else if (flags & MAP_32BIT) { /* * For MAP_32BIT, override the hint if it is too high and * do not bother moving the mapping past the heap (since * the heap is usually above 2GB). */ if (addr + size > MAP_32BIT_MAX_ADDR) addr = 0; -#endif } else { /* * XXX for non-fixed mappings where no hint is provided or * the hint would fall in the potential heap space, * place it after the end of the largest possible heap. * * For anonymous mappings within the address space of the * calling process, the absence of a hint is handled at a * lower level in order to implement different clustering * strategies for ASLR. */ if (((flags & MAP_ANON) == 0 && addr == 0) || (addr >= round_page((vm_offset_t)vms->vm_taddr) && addr < round_page((vm_offset_t)vms->vm_daddr + lim_max(td, RLIMIT_DATA)))) addr = round_page((vm_offset_t)vms->vm_daddr + lim_max(td, RLIMIT_DATA)); } if (len == 0) { /* * Return success without mapping anything for old * binaries that request a page-aligned mapping of * length 0. For modern binaries, this function * returns an error earlier. */ error = 0; } else if ((flags & MAP_GUARD) != 0) { error = vm_mmap_object(&vms->vm_map, &addr, size, VM_PROT_NONE, VM_PROT_NONE, flags, NULL, pos, FALSE, td); } else if ((flags & MAP_ANON) != 0) { /* * Mapping blank space is trivial. * * This relies on VM_PROT_* matching PROT_*. */ error = vm_mmap_object(&vms->vm_map, &addr, size, prot, max_prot, flags, NULL, pos, FALSE, td); } else { /* * Mapping file, get fp for validation and don't let the * descriptor disappear on us if we block. Check capability * rights, but also return the maximum rights to be combined * with maxprot later. */ cap_rights_init_one(&rights, CAP_MMAP); if (prot & PROT_READ) cap_rights_set_one(&rights, CAP_MMAP_R); if ((flags & MAP_SHARED) != 0) { if (prot & PROT_WRITE) cap_rights_set_one(&rights, CAP_MMAP_W); } if (prot & PROT_EXEC) cap_rights_set_one(&rights, CAP_MMAP_X); error = fget_mmap(td, fd, &rights, &cap_maxprot, &fp); if (error != 0) goto done; if ((flags & (MAP_SHARED | MAP_PRIVATE)) == 0 && p->p_osrel >= P_OSREL_MAP_FSTRICT) { error = EINVAL; goto done; } if (check_fp_fn != NULL) { error = check_fp_fn(fp, prot, max_prot & cap_maxprot, flags); if (error != 0) goto done; } if (fp->f_ops == &shm_ops && shm_largepage(fp->f_data)) addr = orig_addr; /* This relies on VM_PROT_* matching PROT_*. */ error = fo_mmap(fp, &vms->vm_map, &addr, size, prot, max_prot & cap_maxprot, flags, pos, td); } if (error == 0) td->td_retval[0] = addr + pageoff; done: if (fp) fdrop(fp, td); return (error); } #if defined(COMPAT_FREEBSD6) int freebsd6_mmap(struct thread *td, struct freebsd6_mmap_args *uap) { return (kern_mmap(td, &(struct mmap_req){ .mr_hint = (uintptr_t)uap->addr, .mr_len = uap->len, .mr_prot = uap->prot, .mr_flags = uap->flags, .mr_fd = uap->fd, .mr_pos = uap->pos, })); } #endif #ifdef COMPAT_43 #ifndef _SYS_SYSPROTO_H_ struct ommap_args { caddr_t addr; int len; int prot; int flags; int fd; long pos; }; #endif int ommap(struct thread *td, struct ommap_args *uap) { return (kern_ommap(td, (uintptr_t)uap->addr, uap->len, uap->prot, uap->flags, uap->fd, uap->pos)); } int kern_ommap(struct thread *td, uintptr_t hint, int len, int oprot, int oflags, int fd, long pos) { static const char cvtbsdprot[8] = { 0, PROT_EXEC, PROT_WRITE, PROT_EXEC | PROT_WRITE, PROT_READ, PROT_EXEC | PROT_READ, PROT_WRITE | PROT_READ, PROT_EXEC | PROT_WRITE | PROT_READ, }; int flags, prot; if (len < 0) return (EINVAL); #define OMAP_ANON 0x0002 #define OMAP_COPY 0x0020 #define OMAP_SHARED 0x0010 #define OMAP_FIXED 0x0100 prot = cvtbsdprot[oprot & 0x7]; #if (defined(COMPAT_FREEBSD32) && defined(__amd64__)) || defined(__i386__) if (i386_read_exec && SV_PROC_FLAG(td->td_proc, SV_ILP32) && prot != 0) prot |= PROT_EXEC; #endif flags = 0; if (oflags & OMAP_ANON) flags |= MAP_ANON; if (oflags & OMAP_COPY) flags |= MAP_COPY; if (oflags & OMAP_SHARED) flags |= MAP_SHARED; else flags |= MAP_PRIVATE; if (oflags & OMAP_FIXED) flags |= MAP_FIXED; return (kern_mmap(td, &(struct mmap_req){ .mr_hint = hint, .mr_len = len, .mr_prot = prot, .mr_flags = flags, .mr_fd = fd, .mr_pos = pos, })); } #endif /* COMPAT_43 */ #ifndef _SYS_SYSPROTO_H_ struct msync_args { void *addr; size_t len; int flags; }; #endif int sys_msync(struct thread *td, struct msync_args *uap) { return (kern_msync(td, (uintptr_t)uap->addr, uap->len, uap->flags)); } int kern_msync(struct thread *td, uintptr_t addr0, size_t size, int flags) { vm_offset_t addr; vm_size_t pageoff; vm_map_t map; int rv; addr = addr0; pageoff = (addr & PAGE_MASK); addr -= pageoff; size += pageoff; size = (vm_size_t) round_page(size); if (addr + size < addr) return (EINVAL); if ((flags & (MS_ASYNC|MS_INVALIDATE)) == (MS_ASYNC|MS_INVALIDATE)) return (EINVAL); map = &td->td_proc->p_vmspace->vm_map; /* * Clean the pages and interpret the return value. */ rv = vm_map_sync(map, addr, addr + size, (flags & MS_ASYNC) == 0, (flags & MS_INVALIDATE) != 0); switch (rv) { case KERN_SUCCESS: return (0); case KERN_INVALID_ADDRESS: return (ENOMEM); case KERN_INVALID_ARGUMENT: return (EBUSY); case KERN_FAILURE: return (EIO); default: return (EINVAL); } } #ifndef _SYS_SYSPROTO_H_ struct munmap_args { void *addr; size_t len; }; #endif int sys_munmap(struct thread *td, struct munmap_args *uap) { return (kern_munmap(td, (uintptr_t)uap->addr, uap->len)); } int kern_munmap(struct thread *td, uintptr_t addr0, size_t size) { #ifdef HWPMC_HOOKS struct pmckern_map_out pkm; vm_map_entry_t entry; bool pmc_handled; #endif vm_offset_t addr, end; vm_size_t pageoff; vm_map_t map; int rv; if (size == 0) return (EINVAL); addr = addr0; pageoff = (addr & PAGE_MASK); addr -= pageoff; size += pageoff; size = (vm_size_t) round_page(size); end = addr + size; map = &td->td_proc->p_vmspace->vm_map; if (!vm_map_range_valid(map, addr, end)) return (EINVAL); vm_map_lock(map); #ifdef HWPMC_HOOKS pmc_handled = false; if (PMC_HOOK_INSTALLED(PMC_FN_MUNMAP)) { pmc_handled = true; /* * Inform hwpmc if the address range being unmapped contains * an executable region. */ pkm.pm_address = (uintptr_t) NULL; if (vm_map_lookup_entry(map, addr, &entry)) { for (; entry->start < end; entry = vm_map_entry_succ(entry)) { if (vm_map_check_protection(map, entry->start, entry->end, VM_PROT_EXECUTE) == TRUE) { pkm.pm_address = (uintptr_t) addr; pkm.pm_size = (size_t) size; break; } } } } #endif rv = vm_map_delete(map, addr, end); #ifdef HWPMC_HOOKS if (rv == KERN_SUCCESS && __predict_false(pmc_handled)) { /* downgrade the lock to prevent a LOR with the pmc-sx lock */ vm_map_lock_downgrade(map); if (pkm.pm_address != (uintptr_t) NULL) PMC_CALL_HOOK(td, PMC_FN_MUNMAP, (void *) &pkm); vm_map_unlock_read(map); } else #endif vm_map_unlock(map); return (vm_mmap_to_errno(rv)); } #ifndef _SYS_SYSPROTO_H_ struct mprotect_args { const void *addr; size_t len; int prot; }; #endif int sys_mprotect(struct thread *td, struct mprotect_args *uap) { return (kern_mprotect(td, (uintptr_t)uap->addr, uap->len, uap->prot, 0)); } int kern_mprotect(struct thread *td, uintptr_t addr0, size_t size, int prot, int flags) { vm_offset_t addr; vm_size_t pageoff; int vm_error, max_prot; addr = addr0; if ((prot & ~(_PROT_ALL | PROT_MAX(_PROT_ALL))) != 0) return (EINVAL); max_prot = PROT_MAX_EXTRACT(prot); prot = PROT_EXTRACT(prot); pageoff = (addr & PAGE_MASK); addr -= pageoff; size += pageoff; size = (vm_size_t) round_page(size); #ifdef COMPAT_FREEBSD32 if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) { if (((addr + size) & 0xffffffff) < addr) return (EINVAL); } else #endif if (addr + size < addr) return (EINVAL); flags |= VM_MAP_PROTECT_SET_PROT; if (max_prot != 0) flags |= VM_MAP_PROTECT_SET_MAXPROT; vm_error = vm_map_protect(&td->td_proc->p_vmspace->vm_map, addr, addr + size, prot, max_prot, flags); switch (vm_error) { case KERN_SUCCESS: return (0); case KERN_PROTECTION_FAILURE: return (EACCES); case KERN_RESOURCE_SHORTAGE: return (ENOMEM); case KERN_OUT_OF_BOUNDS: return (ENOTSUP); } return (EINVAL); } #ifndef _SYS_SYSPROTO_H_ struct minherit_args { void *addr; size_t len; int inherit; }; #endif int sys_minherit(struct thread *td, struct minherit_args *uap) { return (kern_minherit(td, (uintptr_t)uap->addr, uap->len, uap->inherit)); } int kern_minherit(struct thread *td, uintptr_t addr0, size_t len, int inherit0) { vm_offset_t addr; vm_size_t size, pageoff; vm_inherit_t inherit; addr = (vm_offset_t)addr0; size = len; inherit = inherit0; pageoff = (addr & PAGE_MASK); addr -= pageoff; size += pageoff; size = (vm_size_t) round_page(size); if (addr + size < addr) return (EINVAL); switch (vm_map_inherit(&td->td_proc->p_vmspace->vm_map, addr, addr + size, inherit)) { case KERN_SUCCESS: return (0); case KERN_PROTECTION_FAILURE: return (EACCES); } return (EINVAL); } #ifndef _SYS_SYSPROTO_H_ struct madvise_args { void *addr; size_t len; int behav; }; #endif int sys_madvise(struct thread *td, struct madvise_args *uap) { return (kern_madvise(td, (uintptr_t)uap->addr, uap->len, uap->behav)); } int kern_madvise(struct thread *td, uintptr_t addr0, size_t len, int behav) { vm_map_t map; vm_offset_t addr, end, start; int flags; /* * Check for our special case, advising the swap pager we are * "immortal." */ if (behav == MADV_PROTECT) { flags = PPROT_SET; return (kern_procctl(td, P_PID, td->td_proc->p_pid, PROC_SPROTECT, &flags)); } /* * Check for illegal addresses. Watch out for address wrap... Note * that VM_*_ADDRESS are not constants due to casts (argh). */ map = &td->td_proc->p_vmspace->vm_map; addr = addr0; if (!vm_map_range_valid(map, addr, addr + len)) return (EINVAL); /* * Since this routine is only advisory, we default to conservative * behavior. */ start = trunc_page(addr); end = round_page(addr + len); /* * vm_map_madvise() checks for illegal values of behav. */ return (vm_map_madvise(map, start, end, behav)); } #ifndef _SYS_SYSPROTO_H_ struct mincore_args { const void *addr; size_t len; char *vec; }; #endif int sys_mincore(struct thread *td, struct mincore_args *uap) { return (kern_mincore(td, (uintptr_t)uap->addr, uap->len, uap->vec)); } int kern_mincore(struct thread *td, uintptr_t addr0, size_t len, char *vec) { pmap_t pmap; vm_map_t map; vm_map_entry_t current, entry; vm_object_t object; vm_offset_t addr, cend, end, first_addr; vm_paddr_t pa; vm_page_t m; vm_pindex_t pindex; int error, lastvecindex, mincoreinfo, vecindex; unsigned int timestamp; /* * Make sure that the addresses presented are valid for user * mode. */ first_addr = addr = trunc_page(addr0); end = round_page(addr0 + len); map = &td->td_proc->p_vmspace->vm_map; if (end > vm_map_max(map) || end < addr) return (ENOMEM); pmap = vmspace_pmap(td->td_proc->p_vmspace); vm_map_lock_read(map); RestartScan: timestamp = map->timestamp; if (!vm_map_lookup_entry(map, addr, &entry)) { vm_map_unlock_read(map); return (ENOMEM); } /* * Do this on a map entry basis so that if the pages are not * in the current processes address space, we can easily look * up the pages elsewhere. */ lastvecindex = -1; while (entry->start < end) { /* * check for contiguity */ current = entry; entry = vm_map_entry_succ(current); if (current->end < end && entry->start > current->end) { vm_map_unlock_read(map); return (ENOMEM); } /* * ignore submaps (for now) or null objects */ if ((current->eflags & MAP_ENTRY_IS_SUB_MAP) || current->object.vm_object == NULL) continue; /* * limit this scan to the current map entry and the * limits for the mincore call */ if (addr < current->start) addr = current->start; cend = current->end; if (cend > end) cend = end; for (; addr < cend; addr += PAGE_SIZE) { /* * Check pmap first, it is likely faster, also * it can provide info as to whether we are the * one referencing or modifying the page. */ m = NULL; object = NULL; retry: pa = 0; mincoreinfo = pmap_mincore(pmap, addr, &pa); if (mincore_mapped) { /* * We only care about this pmap's * mapping of the page, if any. */ ; } else if (pa != 0) { /* * The page is mapped by this process but not * both accessed and modified. It is also * managed. Acquire the object lock so that * other mappings might be examined. The page's * identity may change at any point before its * object lock is acquired, so re-validate if * necessary. */ m = PHYS_TO_VM_PAGE(pa); while (object == NULL || m->object != object) { if (object != NULL) VM_OBJECT_WUNLOCK(object); object = atomic_load_ptr(&m->object); if (object == NULL) goto retry; VM_OBJECT_WLOCK(object); } if (pa != pmap_extract(pmap, addr)) goto retry; KASSERT(vm_page_all_valid(m), ("mincore: page %p is mapped but invalid", m)); } else if (mincoreinfo == 0) { /* * The page is not mapped by this process. If * the object implements managed pages, then * determine if the page is resident so that * the mappings might be examined. */ if (current->object.vm_object != object) { if (object != NULL) VM_OBJECT_WUNLOCK(object); object = current->object.vm_object; VM_OBJECT_WLOCK(object); } if ((object->flags & OBJ_SWAP) != 0 || object->type == OBJT_VNODE) { pindex = OFF_TO_IDX(current->offset + (addr - current->start)); m = vm_page_lookup(object, pindex); if (m != NULL && vm_page_none_valid(m)) m = NULL; if (m != NULL) mincoreinfo = MINCORE_INCORE; } } if (m != NULL) { VM_OBJECT_ASSERT_WLOCKED(m->object); /* Examine other mappings of the page. */ if (m->dirty == 0 && pmap_is_modified(m)) vm_page_dirty(m); if (m->dirty != 0) mincoreinfo |= MINCORE_MODIFIED_OTHER; /* * The first test for PGA_REFERENCED is an * optimization. The second test is * required because a concurrent pmap * operation could clear the last reference * and set PGA_REFERENCED before the call to * pmap_is_referenced(). */ if ((m->a.flags & PGA_REFERENCED) != 0 || pmap_is_referenced(m) || (m->a.flags & PGA_REFERENCED) != 0) mincoreinfo |= MINCORE_REFERENCED_OTHER; } if (object != NULL) VM_OBJECT_WUNLOCK(object); /* * subyte may page fault. In case it needs to modify * the map, we release the lock. */ vm_map_unlock_read(map); /* * calculate index into user supplied byte vector */ vecindex = atop(addr - first_addr); /* * If we have skipped map entries, we need to make sure that * the byte vector is zeroed for those skipped entries. */ while ((lastvecindex + 1) < vecindex) { ++lastvecindex; error = subyte(vec + lastvecindex, 0); if (error) { error = EFAULT; goto done2; } } /* * Pass the page information to the user */ error = subyte(vec + vecindex, mincoreinfo); if (error) { error = EFAULT; goto done2; } /* * If the map has changed, due to the subyte, the previous * output may be invalid. */ vm_map_lock_read(map); if (timestamp != map->timestamp) goto RestartScan; lastvecindex = vecindex; } } /* * subyte may page fault. In case it needs to modify * the map, we release the lock. */ vm_map_unlock_read(map); /* * Zero the last entries in the byte vector. */ vecindex = atop(end - first_addr); while ((lastvecindex + 1) < vecindex) { ++lastvecindex; error = subyte(vec + lastvecindex, 0); if (error) { error = EFAULT; goto done2; } } /* * If the map has changed, due to the subyte, the previous * output may be invalid. */ vm_map_lock_read(map); if (timestamp != map->timestamp) goto RestartScan; vm_map_unlock_read(map); done2: return (error); } #ifndef _SYS_SYSPROTO_H_ struct mlock_args { const void *addr; size_t len; }; #endif int sys_mlock(struct thread *td, struct mlock_args *uap) { return (kern_mlock(td->td_proc, td->td_ucred, __DECONST(uintptr_t, uap->addr), uap->len)); } int kern_mlock(struct proc *proc, struct ucred *cred, uintptr_t addr0, size_t len) { vm_offset_t addr, end, last, start; vm_size_t npages, size; vm_map_t map; unsigned long nsize; int error; error = priv_check_cred(cred, PRIV_VM_MLOCK); if (error) return (error); addr = addr0; size = len; last = addr + size; start = trunc_page(addr); end = round_page(last); if (last < addr || end < addr) return (EINVAL); npages = atop(end - start); if (npages > vm_page_max_user_wired) return (ENOMEM); map = &proc->p_vmspace->vm_map; PROC_LOCK(proc); nsize = ptoa(npages + pmap_wired_count(map->pmap)); if (nsize > lim_cur_proc(proc, RLIMIT_MEMLOCK)) { PROC_UNLOCK(proc); return (ENOMEM); } PROC_UNLOCK(proc); #ifdef RACCT if (racct_enable) { PROC_LOCK(proc); error = racct_set(proc, RACCT_MEMLOCK, nsize); PROC_UNLOCK(proc); if (error != 0) return (ENOMEM); } #endif error = vm_map_wire(map, start, end, VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); #ifdef RACCT if (racct_enable && error != KERN_SUCCESS) { PROC_LOCK(proc); racct_set(proc, RACCT_MEMLOCK, ptoa(pmap_wired_count(map->pmap))); PROC_UNLOCK(proc); } #endif switch (error) { case KERN_SUCCESS: return (0); case KERN_INVALID_ARGUMENT: return (EINVAL); default: return (ENOMEM); } } #ifndef _SYS_SYSPROTO_H_ struct mlockall_args { int how; }; #endif int sys_mlockall(struct thread *td, struct mlockall_args *uap) { vm_map_t map; int error; map = &td->td_proc->p_vmspace->vm_map; error = priv_check(td, PRIV_VM_MLOCK); if (error) return (error); if ((uap->how == 0) || ((uap->how & ~(MCL_CURRENT|MCL_FUTURE)) != 0)) return (EINVAL); /* * If wiring all pages in the process would cause it to exceed * a hard resource limit, return ENOMEM. */ if (!old_mlock && uap->how & MCL_CURRENT) { if (map->size > lim_cur(td, RLIMIT_MEMLOCK)) return (ENOMEM); } #ifdef RACCT if (racct_enable) { PROC_LOCK(td->td_proc); error = racct_set(td->td_proc, RACCT_MEMLOCK, map->size); PROC_UNLOCK(td->td_proc); if (error != 0) return (ENOMEM); } #endif if (uap->how & MCL_FUTURE) { vm_map_lock(map); vm_map_modflags(map, MAP_WIREFUTURE, 0); vm_map_unlock(map); error = 0; } if (uap->how & MCL_CURRENT) { /* * P1003.1-2001 mandates that all currently mapped pages * will be memory resident and locked (wired) upon return * from mlockall(). vm_map_wire() will wire pages, by * calling vm_fault_wire() for each page in the region. */ error = vm_map_wire(map, vm_map_min(map), vm_map_max(map), VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK); if (error == KERN_SUCCESS) error = 0; else if (error == KERN_RESOURCE_SHORTAGE) error = ENOMEM; else error = EAGAIN; } #ifdef RACCT if (racct_enable && error != KERN_SUCCESS) { PROC_LOCK(td->td_proc); racct_set(td->td_proc, RACCT_MEMLOCK, ptoa(pmap_wired_count(map->pmap))); PROC_UNLOCK(td->td_proc); } #endif return (error); } #ifndef _SYS_SYSPROTO_H_ struct munlockall_args { register_t dummy; }; #endif int sys_munlockall(struct thread *td, struct munlockall_args *uap) { vm_map_t map; int error; map = &td->td_proc->p_vmspace->vm_map; error = priv_check(td, PRIV_VM_MUNLOCK); if (error) return (error); /* Clear the MAP_WIREFUTURE flag from this vm_map. */ vm_map_lock(map); vm_map_modflags(map, 0, MAP_WIREFUTURE); vm_map_unlock(map); /* Forcibly unwire all pages. */ error = vm_map_unwire(map, vm_map_min(map), vm_map_max(map), VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK); #ifdef RACCT if (racct_enable && error == KERN_SUCCESS) { PROC_LOCK(td->td_proc); racct_set(td->td_proc, RACCT_MEMLOCK, 0); PROC_UNLOCK(td->td_proc); } #endif return (error); } #ifndef _SYS_SYSPROTO_H_ struct munlock_args { const void *addr; size_t len; }; #endif int sys_munlock(struct thread *td, struct munlock_args *uap) { return (kern_munlock(td, (uintptr_t)uap->addr, uap->len)); } int kern_munlock(struct thread *td, uintptr_t addr0, size_t size) { vm_offset_t addr, end, last, start; #ifdef RACCT vm_map_t map; #endif int error; error = priv_check(td, PRIV_VM_MUNLOCK); if (error) return (error); addr = addr0; last = addr + size; start = trunc_page(addr); end = round_page(last); if (last < addr || end < addr) return (EINVAL); error = vm_map_unwire(&td->td_proc->p_vmspace->vm_map, start, end, VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); #ifdef RACCT if (racct_enable && error == KERN_SUCCESS) { PROC_LOCK(td->td_proc); map = &td->td_proc->p_vmspace->vm_map; racct_set(td->td_proc, RACCT_MEMLOCK, ptoa(pmap_wired_count(map->pmap))); PROC_UNLOCK(td->td_proc); } #endif return (error == KERN_SUCCESS ? 0 : ENOMEM); } /* * vm_mmap_vnode() * * Helper function for vm_mmap. Perform sanity check specific for mmap * operations on vnodes. */ int vm_mmap_vnode(struct thread *td, vm_size_t objsize, vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp, struct vnode *vp, vm_ooffset_t *foffp, vm_object_t *objp, boolean_t *writecounted) { struct vattr va; vm_object_t obj; vm_ooffset_t foff; struct ucred *cred; int error, flags; bool writex; cred = td->td_ucred; writex = (*maxprotp & VM_PROT_WRITE) != 0 && (*flagsp & MAP_SHARED) != 0; if ((error = vget(vp, LK_SHARED)) != 0) return (error); AUDIT_ARG_VNODE1(vp); foff = *foffp; flags = *flagsp; obj = vp->v_object; if (vp->v_type == VREG) { /* * Get the proper underlying object */ if (obj == NULL) { error = EINVAL; goto done; } if (obj->type == OBJT_VNODE && obj->handle != vp) { vput(vp); vp = (struct vnode *)obj->handle; /* * Bypass filesystems obey the mpsafety of the * underlying fs. Tmpfs never bypasses. */ error = vget(vp, LK_SHARED); if (error != 0) return (error); } if (writex) { *writecounted = TRUE; vm_pager_update_writecount(obj, 0, objsize); } } else { error = EINVAL; goto done; } if ((error = VOP_GETATTR(vp, &va, cred))) goto done; #ifdef MAC /* This relies on VM_PROT_* matching PROT_*. */ error = mac_vnode_check_mmap(cred, vp, (int)prot, flags); if (error != 0) goto done; #endif if ((flags & MAP_SHARED) != 0) { if ((va.va_flags & (SF_SNAPSHOT|IMMUTABLE|APPEND)) != 0) { if (prot & VM_PROT_WRITE) { error = EPERM; goto done; } *maxprotp &= ~VM_PROT_WRITE; } } /* * If it is a regular file without any references * we do not need to sync it. * Adjust object size to be the size of actual file. */ objsize = round_page(va.va_size); if (va.va_nlink == 0) flags |= MAP_NOSYNC; if (obj->type == OBJT_VNODE) { obj = vm_pager_allocate(OBJT_VNODE, vp, objsize, prot, foff, cred); if (obj == NULL) { error = ENOMEM; goto done; } } else { KASSERT((obj->flags & OBJ_SWAP) != 0, ("wrong object type")); vm_object_reference(obj); #if VM_NRESERVLEVEL > 0 if ((obj->flags & OBJ_COLORED) == 0) { VM_OBJECT_WLOCK(obj); vm_object_color(obj, 0); VM_OBJECT_WUNLOCK(obj); } #endif } *objp = obj; *flagsp = flags; VOP_MMAPPED(vp); done: if (error != 0 && *writecounted) { *writecounted = FALSE; vm_pager_update_writecount(obj, objsize, 0); } vput(vp); return (error); } /* * vm_mmap_cdev() * * Helper function for vm_mmap. Perform sanity check specific for mmap * operations on cdevs. */ int vm_mmap_cdev(struct thread *td, vm_size_t objsize, vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp, struct cdev *cdev, struct cdevsw *dsw, vm_ooffset_t *foff, vm_object_t *objp) { vm_object_t obj; int error, flags; flags = *flagsp; if (dsw->d_flags & D_MMAP_ANON) { *objp = NULL; *foff = 0; *maxprotp = VM_PROT_ALL; *flagsp |= MAP_ANON; return (0); } /* * cdevs do not provide private mappings of any kind. */ if ((*maxprotp & VM_PROT_WRITE) == 0 && (prot & VM_PROT_WRITE) != 0) return (EACCES); if (flags & (MAP_PRIVATE|MAP_COPY)) return (EINVAL); /* * Force device mappings to be shared. */ flags |= MAP_SHARED; #ifdef MAC_XXX error = mac_cdev_check_mmap(td->td_ucred, cdev, (int)prot); if (error != 0) return (error); #endif /* * First, try d_mmap_single(). If that is not implemented * (returns ENODEV), fall back to using the device pager. * Note that d_mmap_single() must return a reference to the * object (it needs to bump the reference count of the object * it returns somehow). * * XXX assumes VM_PROT_* == PROT_* */ error = dsw->d_mmap_single(cdev, foff, objsize, objp, (int)prot); if (error != ENODEV) return (error); obj = vm_pager_allocate(OBJT_DEVICE, cdev, objsize, prot, *foff, td->td_ucred); if (obj == NULL) return (EINVAL); *objp = obj; *flagsp = flags; return (0); } int vm_mmap(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot, vm_prot_t maxprot, int flags, objtype_t handle_type, void *handle, vm_ooffset_t foff) { vm_object_t object; struct thread *td = curthread; int error; boolean_t writecounted; if (size == 0) return (EINVAL); size = round_page(size); object = NULL; writecounted = FALSE; switch (handle_type) { case OBJT_DEVICE: { struct cdevsw *dsw; struct cdev *cdev; int ref; cdev = handle; dsw = dev_refthread(cdev, &ref); if (dsw == NULL) return (ENXIO); error = vm_mmap_cdev(td, size, prot, &maxprot, &flags, cdev, dsw, &foff, &object); dev_relthread(cdev, ref); break; } case OBJT_VNODE: error = vm_mmap_vnode(td, size, prot, &maxprot, &flags, handle, &foff, &object, &writecounted); break; default: error = EINVAL; break; } if (error) return (error); error = vm_mmap_object(map, addr, size, prot, maxprot, flags, object, foff, writecounted, td); if (error != 0 && object != NULL) { /* * If this mapping was accounted for in the vnode's * writecount, then undo that now. */ if (writecounted) vm_pager_release_writecount(object, 0, size); vm_object_deallocate(object); } return (error); } int kern_mmap_racct_check(struct thread *td, vm_map_t map, vm_size_t size) { int error; RACCT_PROC_LOCK(td->td_proc); if (map->size + size > lim_cur(td, RLIMIT_VMEM)) { RACCT_PROC_UNLOCK(td->td_proc); return (ENOMEM); } if (racct_set(td->td_proc, RACCT_VMEM, map->size + size)) { RACCT_PROC_UNLOCK(td->td_proc); return (ENOMEM); } if (!old_mlock && map->flags & MAP_WIREFUTURE) { if (ptoa(pmap_wired_count(map->pmap)) + size > lim_cur(td, RLIMIT_MEMLOCK)) { racct_set_force(td->td_proc, RACCT_VMEM, map->size); RACCT_PROC_UNLOCK(td->td_proc); return (ENOMEM); } error = racct_set(td->td_proc, RACCT_MEMLOCK, ptoa(pmap_wired_count(map->pmap)) + size); if (error != 0) { racct_set_force(td->td_proc, RACCT_VMEM, map->size); RACCT_PROC_UNLOCK(td->td_proc); return (error); } } RACCT_PROC_UNLOCK(td->td_proc); return (0); } /* * Internal version of mmap that maps a specific VM object into an * map. Called by mmap for MAP_ANON, vm_mmap, shm_mmap, and vn_mmap. */ int vm_mmap_object(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot, vm_prot_t maxprot, int flags, vm_object_t object, vm_ooffset_t foff, boolean_t writecounted, struct thread *td) { vm_offset_t default_addr, max_addr; int docow, error, findspace, rv; bool curmap, fitit; curmap = map == &td->td_proc->p_vmspace->vm_map; if (curmap) { error = kern_mmap_racct_check(td, map, size); if (error != 0) return (error); } /* * We currently can only deal with page aligned file offsets. * The mmap() system call already enforces this by subtracting * the page offset from the file offset, but checking here * catches errors in device drivers (e.g. d_single_mmap() * callbacks) and other internal mapping requests (such as in * exec). */ if (foff & PAGE_MASK) return (EINVAL); if ((flags & MAP_FIXED) == 0) { fitit = true; *addr = round_page(*addr); } else { if (*addr != trunc_page(*addr)) return (EINVAL); fitit = false; } if (flags & MAP_ANON) { if (object != NULL || foff != 0) return (EINVAL); docow = 0; } else if (flags & MAP_PREFAULT_READ) docow = MAP_PREFAULT; else docow = MAP_PREFAULT_PARTIAL; if ((flags & (MAP_ANON|MAP_SHARED)) == 0) docow |= MAP_COPY_ON_WRITE; if (flags & MAP_NOSYNC) docow |= MAP_DISABLE_SYNCER; if (flags & MAP_NOCORE) docow |= MAP_DISABLE_COREDUMP; /* Shared memory is also shared with children. */ if (flags & MAP_SHARED) docow |= MAP_INHERIT_SHARE; if (writecounted) docow |= MAP_WRITECOUNT; if (flags & MAP_STACK) { if (object != NULL) return (EINVAL); docow |= MAP_STACK_GROWS_DOWN; } if ((flags & MAP_EXCL) != 0) docow |= MAP_CHECK_EXCL; if ((flags & MAP_GUARD) != 0) docow |= MAP_CREATE_GUARD; if (fitit) { if ((flags & MAP_ALIGNMENT_MASK) == MAP_ALIGNED_SUPER) findspace = VMFS_SUPER_SPACE; else if ((flags & MAP_ALIGNMENT_MASK) != 0) findspace = VMFS_ALIGNED_SPACE(flags >> MAP_ALIGNMENT_SHIFT); else findspace = VMFS_OPTIMAL_SPACE; max_addr = 0; -#ifdef MAP_32BIT if ((flags & MAP_32BIT) != 0) max_addr = MAP_32BIT_MAX_ADDR; -#endif if (curmap) { default_addr = round_page((vm_offset_t)td->td_proc->p_vmspace-> vm_daddr + lim_max(td, RLIMIT_DATA)); -#ifdef MAP_32BIT if ((flags & MAP_32BIT) != 0) default_addr = 0; -#endif rv = vm_map_find_min(map, object, foff, addr, size, default_addr, max_addr, findspace, prot, maxprot, docow); } else { rv = vm_map_find(map, object, foff, addr, size, max_addr, findspace, prot, maxprot, docow); } } else { rv = vm_map_fixed(map, object, foff, *addr, size, prot, maxprot, docow); } if (rv == KERN_SUCCESS) { /* * If the process has requested that all future mappings * be wired, then heed this. */ if ((map->flags & MAP_WIREFUTURE) != 0) { vm_map_lock(map); if ((map->flags & MAP_WIREFUTURE) != 0) (void)vm_map_wire_locked(map, *addr, *addr + size, VM_MAP_WIRE_USER | ((flags & MAP_STACK) ? VM_MAP_WIRE_HOLESOK : VM_MAP_WIRE_NOHOLES)); vm_map_unlock(map); } } return (vm_mmap_to_errno(rv)); } /* * Translate a Mach VM return code to zero on success or the appropriate errno * on failure. */ int vm_mmap_to_errno(int rv) { switch (rv) { case KERN_SUCCESS: return (0); case KERN_INVALID_ADDRESS: case KERN_NO_SPACE: return (ENOMEM); case KERN_PROTECTION_FAILURE: return (EACCES); default: return (EINVAL); } } diff --git a/tests/sys/vm/Makefile b/tests/sys/vm/Makefile index c460ba9d8db7..a3ea83a33cfd 100644 --- a/tests/sys/vm/Makefile +++ b/tests/sys/vm/Makefile @@ -1,26 +1,21 @@ # $FreeBSD$ PACKAGE= tests TESTSDIR= ${TESTSBASE}/sys/vm ATF_TESTS_C+= mlock_test \ mmap_test \ page_fault_signal \ shared_shadow_inval_test -.if ${MACHINE_ARCH} != "i386" && ${MACHINE} != "arm" && \ - (${MACHINE} != "powerpc" || (${MACHINE_ARCH} != "powerpc" && \ - ${MACHINE_ARCH} != "powerpcspe")) - # MAP_32BIT is only available on 64-bit platforms BINDIR= ${TESTSDIR} ATF_TESTS_SH+= mmap_map_32bit_test PROGS+= mmap_map_32bit_helper -.endif SUBDIR= soxstack TESTS_SUBDIRS+= stack SUBDIR_DEPEND_stack=soxstack .include diff --git a/tests/sys/vm/mmap_map_32bit_helper.c b/tests/sys/vm/mmap_map_32bit_helper.c index 47d4b2c53c20..763bba35d2b9 100644 --- a/tests/sys/vm/mmap_map_32bit_helper.c +++ b/tests/sys/vm/mmap_map_32bit_helper.c @@ -1,51 +1,50 @@ /*- * Copyright (c) 2023 Dmitry Chagin * * SPDX-License-Identifier: BSD-2-Clause */ #include #include #include #include #include -_Static_assert(sizeof(vm_offset_t) >= 8, "Test is not intended for ILP32"); #define MAP_32BIT_MAX_ADDR ((vm_offset_t)1 << 31) int main(void) { size_t pagesize; void *s32; int fd; if ((pagesize = getpagesize()) <= 0) err(1, "getpagesize"); fd = open("/dev/zero", O_RDONLY); if (fd <= 0) err(1, "open failed"); s32 = mmap(NULL, pagesize, PROT_READ, MAP_32BIT | MAP_PRIVATE, fd, 0); if (s32 == MAP_FAILED) err(1, "mmap MAP_32BIT | MAP_PRIVATE failed"); if (((vm_offset_t)s32 + pagesize) > MAP_32BIT_MAX_ADDR) errx(1, "mmap invalid result %p", s32); close(fd); if (munmap(s32, pagesize) != 0) err(1, "munmap failed"); s32 = mmap(NULL, pagesize, PROT_READ | PROT_WRITE, MAP_32BIT | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); if (s32 == MAP_FAILED) err(1, "mmap MAP_32BIT | MAP_ANONYMOUS | MAP_PRIVATE failed"); if (((vm_offset_t)s32 + pagesize) > MAP_32BIT_MAX_ADDR) errx(1, "mmap invalid result %p", s32); if (munmap(s32, pagesize) != 0) err(1, "munmap failed"); exit(0); }