Index: head/sys/compat/freebsd32/syscalls.master =================================================================== --- head/sys/compat/freebsd32/syscalls.master (revision 162535) +++ head/sys/compat/freebsd32/syscalls.master (revision 162536) @@ -1,762 +1,764 @@ $FreeBSD$ ; from: @(#)syscalls.master 8.2 (Berkeley) 1/13/94 ; from: src/sys/kern/syscalls.master 1.107 ; ; System call name/number master file. ; Processed to created init_sysent.c, syscalls.c and syscall.h. ; Columns: number audit type nargs name alt{name,tag,rtyp}/comments ; number system call number, must be in order ; audit the audit event associated with the system call ; A value of AUE_NULL means no auditing, but it also means that ; there is no audit event for the call at this time. For the ; case where the event exists, but we don't want auditing, the ; event should be #defined to AUE_NULL in audit_kevents.h. ; type one of STD, OBSOL, UNIMPL, COMPAT, CPT_NOA, LIBCOMPAT, ; NODEF, NOARGS, NOPROTO, NOIMPL, NOSTD, COMPAT4 ; name psuedo-prototype of syscall routine ; If one of the following alts is different, then all appear: ; altname name of system call if different ; alttag name of args struct tag if different from [o]`name'"_args" ; altrtyp return type if not int (bogus - syscalls always return int) ; for UNIMPL/OBSOL, name continues with comments ; types: ; STD always included ; COMPAT included on COMPAT #ifdef ; COMPAT4 included on COMPAT4 #ifdef (FreeBSD 4 compat) ; LIBCOMPAT included on COMPAT #ifdef, and placed in syscall.h ; OBSOL obsolete, not included in system, only specifies name ; UNIMPL not implemented, placeholder only ; NOSTD implemented but as a lkm that can be statically ; compiled in; sysent entry will be filled with lkmsys ; so the SYSCALL_MODULE macro works ; #ifdef's, etc. may be included, and are copied to the output files. #include #include #include #include #include #include ; Reserved/unimplemented system calls in the range 0-150 inclusive ; are reserved for use in future Berkeley releases. ; Additional system calls implemented in vendor and other ; redistributions should be placed in the reserved range at the end ; of the current calls. 0 AUE_NULL NOPROTO { int nosys(void); } syscall nosys_args int 1 AUE_EXIT NOPROTO { void sys_exit(int rval); } exit \ sys_exit_args void 2 AUE_FORK NOPROTO { int fork(void); } 3 AUE_READ NOPROTO { ssize_t read(int fd, void *buf, \ size_t nbyte); } 4 AUE_WRITE NOPROTO { ssize_t write(int fd, const void *buf, \ size_t nbyte); } 5 AUE_OPEN_RWTC NOPROTO { int open(char *path, int flags, \ int mode); } 6 AUE_CLOSE NOPROTO { int close(int fd); } 7 AUE_WAIT4 STD { int freebsd32_wait4(int pid, int *status, \ int options, struct rusage32 *rusage); } 8 AUE_CREAT OBSOL old creat 9 AUE_LINK NOPROTO { int link(char *path, char *link); } 10 AUE_UNLINK NOPROTO { int unlink(char *path); } 11 AUE_NULL OBSOL execv 12 AUE_CHDIR NOPROTO { int chdir(char *path); } 13 AUE_FCHDIR NOPROTO { int fchdir(int fd); } 14 AUE_MKNOD NOPROTO { int mknod(char *path, int mode, int dev); } 15 AUE_CHMOD NOPROTO { int chmod(char *path, int mode); } 16 AUE_CHOWN NOPROTO { int chown(char *path, int uid, int gid); } 17 AUE_NULL NOPROTO { int obreak(char *nsize); } break \ obreak_args int 18 AUE_GETFSSTAT COMPAT4 { int freebsd32_getfsstat( \ struct statfs32 *buf, long bufsize, \ int flags); } 19 AUE_LSEEK OBSOL olseek 20 AUE_GETPID NOPROTO { pid_t getpid(void); } 21 AUE_MOUNT NOPROTO { int mount(char *type, char *path, \ int flags, caddr_t data); } 22 AUE_UMOUNT NOPROTO { int unmount(char *path, int flags); } 23 AUE_SETUID NOPROTO { int setuid(uid_t uid); } 24 AUE_GETUID NOPROTO { uid_t getuid(void); } 25 AUE_GETEUID NOPROTO { uid_t geteuid(void); } 26 AUE_PTRACE NOPROTO { int ptrace(int req, pid_t pid, \ caddr_t addr, int data); } ; XXX implement 27 AUE_RECVMSG STD { int freebsd32_recvmsg(int s, struct msghdr32 *msg, \ int flags); } 28 AUE_SENDMSG STD { int freebsd32_sendmsg(int s, struct msghdr32 *msg, \ int flags); } 29 AUE_RECVFROM STD { int freebsd32_recvfrom(int s, u_int32_t buf, \ u_int32_t len, int flags, u_int32_t from, \ u_int32_t fromlenaddr); } 30 AUE_ACCEPT NOPROTO { int accept(int s, caddr_t name, \ int *anamelen); } 31 AUE_GETPEERNAME NOPROTO { int getpeername(int fdes, caddr_t asa, \ int *alen); } 32 AUE_GETSOCKNAME NOPROTO { int getsockname(int fdes, caddr_t asa, \ int *alen); } 33 AUE_ACCESS NOPROTO { int access(char *path, int flags); } 34 AUE_CHFLAGS NOPROTO { int chflags(char *path, int flags); } 35 AUE_FCHFLAGS NOPROTO { int fchflags(int fd, int flags); } 36 AUE_SYNC NOPROTO { int sync(void); } 37 AUE_KILL NOPROTO { int kill(int pid, int signum); } 38 AUE_STAT UNIMPL ostat 39 AUE_GETPPID NOPROTO { pid_t getppid(void); } 40 AUE_LSTAT UNIMPL olstat 41 AUE_DUP NOPROTO { int dup(u_int fd); } 42 AUE_PIPE NOPROTO { int pipe(void); } 43 AUE_GETEGID NOPROTO { gid_t getegid(void); } 44 AUE_PROFILE NOPROTO { int profil(caddr_t samples, size_t size, \ size_t offset, u_int scale); } 45 AUE_KTRACE NOPROTO { int ktrace(const char *fname, int ops, \ int facs, int pid); } 46 AUE_SIGACTION COMPAT { int freebsd32_sigaction( int signum, \ struct osigaction32 *nsa, \ struct osigaction32 *osa); } 47 AUE_GETGID NOPROTO { gid_t getgid(void); } 48 AUE_SIGPROCMASK COMPAT { int freebsd32_sigprocmask(int how, \ osigset_t mask); } 49 AUE_GETLOGIN NOPROTO { int getlogin(char *namebuf, \ u_int namelen); } 50 AUE_SETLOGIN NOPROTO { int setlogin(char *namebuf); } 51 AUE_ACCT NOPROTO { int acct(char *path); } 52 AUE_SIGPENDING COMPAT { int freebsd32_sigpending(void); } 53 AUE_SIGALTSTACK STD { int freebsd32_sigaltstack( \ struct sigaltstack32 *ss, \ struct sigaltstack32 *oss); } 54 AUE_IOCTL NOPROTO { int ioctl(int fd, u_long com, \ caddr_t data); } 55 AUE_REBOOT NOPROTO { int reboot(int opt); } 56 AUE_REVOKE NOPROTO { int revoke(char *path); } 57 AUE_SYMLINK NOPROTO { int symlink(char *path, char *link); } 58 AUE_READLINK NOPROTO { int readlink(char *path, char *buf, \ int count); } 59 AUE_EXECVE STD { int freebsd32_execve(char *fname, \ u_int32_t *argv, u_int32_t *envv); } 60 AUE_UMASK NOPROTO { int umask(int newmask); } umask \ umask_args int 61 AUE_CHROOT NOPROTO { int chroot(char *path); } 62 AUE_FSTAT OBSOL ofstat 63 AUE_NULL OBSOL ogetkerninfo 64 AUE_GETPAGESIZE OBSOL ogetpagesize ; XXX implement (not OBSOL at all) 65 AUE_MSYNC NOPROTO { int msync(void *addr, size_t len, \ int flags); } 66 AUE_VFORK NOPROTO { int vfork(void); } 67 AUE_NULL OBSOL vread 68 AUE_NULL OBSOL vwrite 69 AUE_SBRK NOPROTO { int sbrk(int incr); } 70 AUE_SSTK NOPROTO { int sstk(int incr); } 71 AUE_MMAP OBSOL ommap 72 AUE_O_VADVISE NOPROTO { int ovadvise(int anom); } vadvise \ ovadvise_args int 73 AUE_MUNMAP NOPROTO { int munmap(void *addr, size_t len); } 74 AUE_MPROTECT NOPROTO { int mprotect(const void *addr, \ size_t len, int prot); } 75 AUE_MADVISE NOPROTO { int madvise(void *addr, size_t len, \ int behav); } 76 AUE_NULL OBSOL vhangup 77 AUE_NULL OBSOL vlimit 78 AUE_MINCORE NOPROTO { int mincore(const void *addr, size_t len, \ char *vec); } 79 AUE_GETGROUPS NOPROTO { int getgroups(u_int gidsetsize, \ gid_t *gidset); } 80 AUE_SETGROUPS NOPROTO { int setgroups(u_int gidsetsize, \ gid_t *gidset); } 81 AUE_GETPGRP NOPROTO { int getpgrp(void); } 82 AUE_SETPGRP NOPROTO { int setpgid(int pid, int pgid); } 83 AUE_SETITIMER STD { int freebsd32_setitimer(u_int which, \ struct itimerval32 *itv, \ struct itimerval32 *oitv); } 84 AUE_NULL OBSOL owait ; XXX implement 85 AUE_SWAPON OBSOL oswapon 86 AUE_GETITIMER STD { int freebsd32_getitimer(u_int which, \ struct itimerval32 *itv); } 87 AUE_O_GETHOSTNAME OBSOL ogethostname 88 AUE_O_SETHOSTNAME OBSOL osethostname 89 AUE_GETDTABLESIZE NOPROTO { int getdtablesize(void); } 90 AUE_DUP2 NOPROTO { int dup2(u_int from, u_int to); } 91 AUE_NULL UNIMPL getdopt 92 AUE_FCNTL NOPROTO { int fcntl(int fd, int cmd, long arg); } 93 AUE_SELECT STD { int freebsd32_select(int nd, fd_set *in, \ fd_set *ou, fd_set *ex, \ struct timeval32 *tv); } ; XXX need to override for big-endian - little-endian should work fine. 94 AUE_NULL UNIMPL setdopt 95 AUE_FSYNC NOPROTO { int fsync(int fd); } 96 AUE_SETPRIORITY NOPROTO { int setpriority(int which, int who, \ int prio); } 97 AUE_SOCKET NOPROTO { int socket(int domain, int type, \ int protocol); } 98 AUE_CONNECT NOPROTO { int connect(int s, caddr_t name, \ int namelen); } 99 AUE_NULL OBSOL oaccept 100 AUE_GETPRIORITY NOPROTO { int getpriority(int which, int who); } 101 AUE_NULL OBSOL osend 102 AUE_NULL OBSOL orecv 103 AUE_NULL OBSOL osigreturn 104 AUE_BIND NOPROTO { int bind(int s, caddr_t name, \ int namelen); } 105 AUE_SETSOCKOPT NOPROTO { int setsockopt(int s, int level, \ int name, caddr_t val, int valsize); } 106 AUE_LISTEN NOPROTO { int listen(int s, int backlog); } 107 AUE_NULL OBSOL vtimes 108 AUE_O_SIGVEC COMPAT { int freebsd32_sigvec(int signum, \ struct sigvec32 *nsv, \ struct sigvec32 *osv); } 109 AUE_O_SIGBLOCK COMPAT { int freebsd32_sigblock(int mask); } 110 AUE_O_SIGSETMASK COMPAT { int freebsd32_sigsetmask( int mask); } 111 AUE_SIGSUSPEND COMPAT { int freebsd32_sigsuspend( int mask); } 112 AUE_O_SIGSTACK COMPAT { int freebsd32_sigstack( \ struct sigstack32 *nss, \ struct sigstack32 *oss); } 113 AUE_NULL OBSOL orecvmsg 114 AUE_NULL OBSOL osendmsg 115 AUE_NULL OBSOL vtrace 116 AUE_GETTIMEOFDAY STD { int freebsd32_gettimeofday( \ struct timeval32 *tp, \ struct timezone *tzp); } 117 AUE_GETRUSAGE STD { int freebsd32_getrusage(int who, \ struct rusage32 *rusage); } 118 AUE_GETSOCKOPT NOPROTO { int getsockopt(int s, int level, \ int name, caddr_t val, int *avalsize); } 119 AUE_NULL UNIMPL resuba (BSD/OS 2.x) 120 AUE_READV STD { int freebsd32_readv(int fd, \ struct iovec32 *iovp, u_int iovcnt); } 121 AUE_WRITEV STD { int freebsd32_writev(int fd, \ struct iovec32 *iovp, u_int iovcnt); } 122 AUE_SETTIMEOFDAY STD { int freebsd32_settimeofday( \ struct timeval32 *tv, \ struct timezone *tzp); } 123 AUE_FCHOWN NOPROTO { int fchown(int fd, int uid, int gid); } 124 AUE_FCHMOD NOPROTO { int fchmod(int fd, int mode); } 125 AUE_RECVFROM OBSOL orecvfrom 126 AUE_SETREUID NOPROTO { int setreuid(int ruid, int euid); } 127 AUE_SETREGID NOPROTO { int setregid(int rgid, int egid); } 128 AUE_RENAME NOPROTO { int rename(char *from, char *to); } 129 AUE_TRUNCATE OBSOL otruncate 130 AUE_FTRUNCATE OBSOL ftruncate 131 AUE_FLOCK NOPROTO { int flock(int fd, int how); } 132 AUE_MKFIFO NOPROTO { int mkfifo(char *path, int mode); } 133 AUE_SENDTO NOPROTO { int sendto(int s, caddr_t buf, \ size_t len, int flags, caddr_t to, \ int tolen); } 134 AUE_SHUTDOWN NOPROTO { int shutdown(int s, int how); } 135 AUE_SOCKETPAIR NOPROTO { int socketpair(int domain, int type, \ int protocol, int *rsv); } 136 AUE_MKDIR NOPROTO { int mkdir(char *path, int mode); } 137 AUE_RMDIR NOPROTO { int rmdir(char *path); } 138 AUE_UTIMES STD { int freebsd32_utimes(char *path, \ struct timeval32 *tptr); } 139 AUE_NULL OBSOL 4.2 sigreturn 140 AUE_ADJTIME STD { int freebsd32_adjtime( \ struct timeval32 *delta, \ struct timeval32 *olddelta); } 141 AUE_GETPEERNAME OBSOL ogetpeername 142 AUE_SYSCTL OBSOL ogethostid 143 AUE_SYSCTL OBSOL sethostid 144 AUE_GETRLIMIT OBSOL getrlimit 145 AUE_SETRLIMIT OBSOL setrlimit 146 AUE_KILLPG OBSOL killpg 147 AUE_SETSID NOPROTO { int setsid(void); } 148 AUE_QUOTACTL NOPROTO { int quotactl(char *path, int cmd, int uid, \ caddr_t arg); } 149 AUE_O_QUOTA OBSOL oquota 150 AUE_GETSOCKNAME OBSOL ogetsockname ; Syscalls 151-180 inclusive are reserved for vendor-specific ; system calls. (This includes various calls added for compatibity ; with other Unix variants.) ; Some of these calls are now supported by BSD... 151 AUE_NULL UNIMPL sem_lock (BSD/OS 2.x) 152 AUE_NULL UNIMPL sem_wakeup (BSD/OS 2.x) 153 AUE_NULL UNIMPL asyncdaemon (BSD/OS 2.x) 154 AUE_NULL UNIMPL nosys ; 155 is initialized by the NFS code, if present. ; XXX this is a problem!!! 155 AUE_NFS_SVC UNIMPL nfssvc 156 AUE_GETDIRENTRIES OBSOL ogetdirentries 157 AUE_STATFS COMPAT4 { int freebsd32_statfs(char *path, \ struct statfs32 *buf); } 158 AUE_FSTATFS COMPAT4 { int freebsd32_fstatfs(int fd, \ struct statfs32 *buf); } 159 AUE_NULL UNIMPL nosys 160 AUE_LGETFH UNIMPL lgetfh 161 AUE_NFS_GETFH NOPROTO { int getfh(char *fname, \ struct fhandle *fhp); } 162 AUE_SYSCTL NOPROTO { int getdomainname(char *domainname, \ int len); } 163 AUE_SYSCTL NOPROTO { int setdomainname(char *domainname, \ int len); } 164 AUE_NULL NOPROTO { int uname(struct utsname *name); } 165 AUE_SYSARCH NOPROTO { int sysarch(int op, char *parms); } 166 AUE_RTPRIO NOPROTO { int rtprio(int function, pid_t pid, \ struct rtprio *rtp); } 167 AUE_NULL UNIMPL nosys 168 AUE_NULL UNIMPL nosys 169 AUE_SEMSYS STD { int freebsd32_semsys(int which, int a2, \ int a3, int a4, int a5); } 170 AUE_MSGSYS STD { int freebsd32_msgsys(int which, int a2, \ int a3, int a4, int a5, int a6); } 171 AUE_SHMSYS STD { int freebsd32_shmsys(uint32_t which, uint32_t a2, \ uint32_t a3, uint32_t a4); } 172 AUE_NULL UNIMPL nosys 173 AUE_PREAD STD { ssize_t freebsd32_pread(int fd, void *buf, \ size_t nbyte, int pad, \ u_int32_t offsetlo, u_int32_t offsethi); } ; XXX note - bigendian is different 174 AUE_PWRITE STD { ssize_t freebsd32_pwrite(int fd, \ const void *buf, size_t nbyte, int pad, \ u_int32_t offsetlo, u_int32_t offsethi); } ; XXX note - bigendian is different 175 AUE_NULL UNIMPL nosys 176 AUE_NTP_ADJTIME NOPROTO { int ntp_adjtime(struct timex *tp); } 177 AUE_NULL UNIMPL sfork (BSD/OS 2.x) 178 AUE_NULL UNIMPL getdescriptor (BSD/OS 2.x) 179 AUE_NULL UNIMPL setdescriptor (BSD/OS 2.x) 180 AUE_NULL UNIMPL nosys ; Syscalls 181-199 are used by/reserved for BSD 181 AUE_SETGID NOPROTO { int setgid(gid_t gid); } 182 AUE_SETEGID NOPROTO { int setegid(gid_t egid); } 183 AUE_SETEUID NOPROTO { int seteuid(uid_t euid); } 184 AUE_NULL UNIMPL lfs_bmapv 185 AUE_NULL UNIMPL lfs_markv 186 AUE_NULL UNIMPL lfs_segclean 187 AUE_NULL UNIMPL lfs_segwait 188 AUE_STAT STD { int freebsd32_stat(char *path, \ struct stat32 *ub); } 189 AUE_FSTAT STD { int freebsd32_fstat(int fd, \ struct stat32 *ub); } 190 AUE_LSTAT STD { int freebsd32_lstat(char *path, \ struct stat32 *ub); } 191 AUE_PATHCONF NOPROTO { int pathconf(char *path, int name); } 192 AUE_FPATHCONF NOPROTO { int fpathconf(int fd, int name); } 193 AUE_NULL UNIMPL nosys 194 AUE_GETRLIMIT NOPROTO { int getrlimit(u_int which, \ struct rlimit *rlp); } getrlimit \ __getrlimit_args int 195 AUE_SETRLIMIT NOPROTO { int setrlimit(u_int which, \ struct rlimit *rlp); } setrlimit \ __setrlimit_args int 196 AUE_GETDIRENTRIES NOPROTO { int getdirentries(int fd, char *buf, \ u_int count, long *basep); } 197 AUE_MMAP STD { caddr_t freebsd32_mmap(caddr_t addr, \ size_t len, int prot, int flags, int fd, \ int pad, u_int32_t poslo, \ u_int32_t poshi); } 198 AUE_NULL NOPROTO { int nosys(void); } __syscall \ __syscall_args int ; XXX note - bigendian is different 199 AUE_LSEEK STD { off_t freebsd32_lseek(int fd, int pad, \ u_int32_t offsetlo, u_int32_t offsethi, \ int whence); } ; XXX note - bigendian is different 200 AUE_TRUNCATE STD { int freebsd32_truncate(char *path, \ int pad, u_int32_t lengthlo, \ u_int32_t lengthhi); } ; XXX note - bigendian is different 201 AUE_FTRUNCATE STD { int freebsd32_ftruncate(int fd, int pad, \ u_int32_t lengthlo, u_int32_t lengthhi); } 202 AUE_SYSCTL STD { int freebsd32_sysctl(int *name, \ u_int namelen, void *old, \ u_int32_t *oldlenp, void *new, \ u_int32_t newlen); } 203 AUE_MLOCK NOPROTO { int mlock(const void *addr, \ size_t len); } 204 AUE_MUNLOCK NOPROTO { int munlock(const void *addr, \ size_t len); } 205 AUE_UNDELETE NOPROTO { int undelete(char *path); } 206 AUE_FUTIMES STD { int freebsd32_futimes(int fd, \ struct timeval32 *tptr); } 207 AUE_GETPGID NOPROTO { int getpgid(pid_t pid); } 208 AUE_NULL UNIMPL newreboot (NetBSD) 209 AUE_POLL NOPROTO { int poll(struct pollfd *fds, u_int nfds, \ int timeout); } ; ; The following are reserved for loadable syscalls ; 210 AUE_NULL UNIMPL 211 AUE_NULL UNIMPL 212 AUE_NULL UNIMPL 213 AUE_NULL UNIMPL 214 AUE_NULL UNIMPL 215 AUE_NULL UNIMPL 216 AUE_NULL UNIMPL 217 AUE_NULL UNIMPL 218 AUE_NULL UNIMPL 219 AUE_NULL UNIMPL ; ; The following were introduced with NetBSD/4.4Lite-2 ; They are initialized by thier respective modules/sysinits ; XXX PROBLEM!! 220 AUE_SEMCTL NOPROTO { int __semctl(int semid, int semnum, \ int cmd, union semun *arg); } 221 AUE_SEMGET NOPROTO { int semget(key_t key, int nsems, \ int semflg); } 222 AUE_SEMOP NOPROTO { int semop(int semid, struct sembuf *sops, \ u_int nsops); } 223 AUE_NULL UNIMPL semconfig 224 AUE_MSGCTL NOPROTO { int msgctl(int msqid, int cmd, \ struct msqid_ds *buf); } 225 AUE_MSGGET NOPROTO { int msgget(key_t key, int msgflg); } 226 AUE_MSGSND NOPROTO { int msgsnd(int msqid, void *msgp, \ size_t msgsz, int msgflg); } 227 AUE_MSGRCV NOPROTO { int msgrcv(int msqid, void *msgp, \ size_t msgsz, long msgtyp, int msgflg); } 228 AUE_SHMAT NOPROTO { int shmat(int shmid, void *shmaddr, \ int shmflg); } 229 AUE_SHMCTL STD { int freebsd32_shmctl(int shmid, int cmd, \ struct shmid_ds *buf); } 230 AUE_SHMDT NOPROTO { int shmdt(void *shmaddr); } 231 AUE_SHMGET NOPROTO { int shmget(key_t key, int size, \ int shmflg); } ; 232 AUE_NULL STD { int freebsd32_clock_gettime(clockid_t clock_id, \ struct timespec32 *tp); } 233 AUE_CLOCK_SETTIME STD { int freebsd32_clock_settime(clockid_t clock_id, \ const struct timespec32 *tp); } 234 AUE_NULL STD { int freebsd32_clock_getres(clockid_t clock_id, \ struct timespec32 *tp); } 235 AUE_NULL UNIMPL timer_create 236 AUE_NULL UNIMPL timer_delete 237 AUE_NULL UNIMPL timer_settime 238 AUE_NULL UNIMPL timer_gettime 239 AUE_NULL UNIMPL timer_getoverrun 240 AUE_NULL STD { int freebsd32_nanosleep( \ const struct timespec32 *rqtp, \ struct timespec32 *rmtp); } 241 AUE_NULL UNIMPL nosys 242 AUE_NULL UNIMPL nosys 243 AUE_NULL UNIMPL nosys 244 AUE_NULL UNIMPL nosys 245 AUE_NULL UNIMPL nosys 246 AUE_NULL UNIMPL nosys 247 AUE_NULL UNIMPL nosys 248 AUE_NULL UNIMPL ntp_gettime 249 AUE_NULL UNIMPL nosys ; syscall numbers initially used in OpenBSD 250 AUE_MINHERIT NOPROTO { int minherit(void *addr, size_t len, \ int inherit); } 251 AUE_RFORK NOPROTO { int rfork(int flags); } 252 AUE_POLL NOPROTO { int openbsd_poll(struct pollfd *fds, \ u_int nfds, int timeout); } 253 AUE_ISSETUGID NOPROTO { int issetugid(void); } 254 AUE_LCHOWN NOPROTO { int lchown(char *path, int uid, int gid); } 255 AUE_NULL UNIMPL nosys 256 AUE_NULL UNIMPL nosys 257 AUE_NULL UNIMPL nosys 258 AUE_NULL UNIMPL nosys 259 AUE_NULL UNIMPL nosys 260 AUE_NULL UNIMPL nosys 261 AUE_NULL UNIMPL nosys 262 AUE_NULL UNIMPL nosys 263 AUE_NULL UNIMPL nosys 264 AUE_NULL UNIMPL nosys 265 AUE_NULL UNIMPL nosys 266 AUE_NULL UNIMPL nosys 267 AUE_NULL UNIMPL nosys 268 AUE_NULL UNIMPL nosys 269 AUE_NULL UNIMPL nosys 270 AUE_NULL UNIMPL nosys 271 AUE_NULL UNIMPL nosys 272 AUE_O_GETDENTS NOPROTO { int getdents(int fd, char *buf, \ size_t count); } 273 AUE_NULL UNIMPL nosys 274 AUE_LCHMOD NOPROTO { int lchmod(char *path, mode_t mode); } 275 AUE_LCHOWN NOPROTO { int lchown(char *path, uid_t uid, \ gid_t gid); } netbsd_lchown \ lchown_args int 276 AUE_LUTIMES STD { int freebsd32_lutimes(char *path, \ struct timeval32 *tptr); } 277 AUE_MSYNC NOPROTO { int msync(void *addr, size_t len, \ int flags); } netbsd_msync msync_args int 278 AUE_STAT NOPROTO { int nstat(char *path, struct nstat *ub); } 279 AUE_FSTAT NOPROTO { int nfstat(int fd, struct nstat *sb); } 280 AUE_LSTAT NOPROTO { int nlstat(char *path, struct nstat *ub); } 281 AUE_NULL UNIMPL nosys 282 AUE_NULL UNIMPL nosys 283 AUE_NULL UNIMPL nosys 284 AUE_NULL UNIMPL nosys 285 AUE_NULL UNIMPL nosys 286 AUE_NULL UNIMPL nosys 287 AUE_NULL UNIMPL nosys 288 AUE_NULL UNIMPL nosys ; 289 and 290 from NetBSD (OpenBSD: 267 and 268) 289 AUE_PREADV STD { ssize_t freebsd32_preadv(int fd, \ struct iovec32 *iovp, \ u_int iovcnt, off_t offset); } ; XXX note - bigendian is different 290 AUE_PWRITEV STD { ssize_t freebsd32_pwritev(int fd, \ struct iovec32 *iovp, \ u_int iovcnt, off_t offset); } ; XXX note - bigendian is different 291 AUE_NULL UNIMPL nosys 292 AUE_NULL UNIMPL nosys 293 AUE_NULL UNIMPL nosys 294 AUE_NULL UNIMPL nosys 295 AUE_NULL UNIMPL nosys 296 AUE_NULL UNIMPL nosys ; XXX 297 is 300 in NetBSD 297 AUE_FHSTATFS COMPAT4 { int freebsd32_fhstatfs( \ const struct fhandle *u_fhp, \ struct statfs32 *buf); } 298 AUE_FHOPEN NOPROTO { int fhopen(const struct fhandle *u_fhp, \ int flags); } 299 AUE_FHSTAT NOPROTO { int fhstat(const struct fhandle *u_fhp, \ struct stat *sb); } ; syscall numbers for FreeBSD 300 AUE_NULL NOPROTO { int modnext(int modid); } 301 AUE_NULL STD { int freebsd32_modstat(int modid, \ struct module_stat32* stat); } 302 AUE_NULL NOPROTO { int modfnext(int modid); } 303 AUE_NULL NOPROTO { int modfind(const char *name); } 304 AUE_MODLOAD NOPROTO { int kldload(const char *file); } 305 AUE_MODUNLOAD NOPROTO { int kldunload(int fileid); } 306 AUE_NULL NOPROTO { int kldfind(const char *file); } 307 AUE_NULL NOPROTO { int kldnext(int fileid); } 308 AUE_NULL NOPROTO { int kldstat(int fileid, \ struct kld_file_stat* stat); } 309 AUE_NULL NOPROTO { int kldfirstmod(int fileid); } 310 AUE_GETSID NOPROTO { int getsid(pid_t pid); } 311 AUE_SETRESUID NOPROTO { int setresuid(uid_t ruid, uid_t euid, \ uid_t suid); } 312 AUE_SETRESGID NOPROTO { int setresgid(gid_t rgid, gid_t egid, \ gid_t sgid); } 313 AUE_NULL OBSOL signanosleep 314 AUE_NULL UNIMPL aio_return 315 AUE_NULL UNIMPL aio_suspend 316 AUE_NULL UNIMPL aio_cancel 317 AUE_NULL UNIMPL aio_error 318 AUE_NULL UNIMPL aio_read 319 AUE_NULL UNIMPL aio_write 320 AUE_NULL UNIMPL lio_listio 321 AUE_NULL NOPROTO { int yield(void); } 322 AUE_NULL OBSOL thr_sleep 323 AUE_NULL OBSOL thr_wakeup 324 AUE_MLOCKALL NOPROTO { int mlockall(int how); } 325 AUE_MUNLOCKALL NOPROTO { int munlockall(void); } 326 AUE_GETCWD NOPROTO { int __getcwd(u_char *buf, u_int buflen); } 327 AUE_NULL NOPROTO { int sched_setparam (pid_t pid, \ const struct sched_param *param); } 328 AUE_NULL NOPROTO { int sched_getparam (pid_t pid, \ struct sched_param *param); } 329 AUE_NULL NOPROTO { int sched_setscheduler (pid_t pid, \ int policy, \ const struct sched_param *param); } 330 AUE_NULL NOPROTO { int sched_getscheduler (pid_t pid); } 331 AUE_NULL NOPROTO { int sched_yield (void); } 332 AUE_NULL NOPROTO { int sched_get_priority_max (int policy); } 333 AUE_NULL NOPROTO { int sched_get_priority_min (int policy); } 334 AUE_NULL NOPROTO { int sched_rr_get_interval (pid_t pid, \ struct timespec *interval); } 335 AUE_NULL NOPROTO { int utrace(const void *addr, size_t len); } ; XXX note - bigendian is different 336 AUE_SENDFILE COMPAT4 { int freebsd32_sendfile(int fd, int s, \ u_int32_t offsetlo, u_int32_t offsethi, \ size_t nbytes, struct sf_hdtr32 *hdtr, \ off_t *sbytes, int flags); } 337 AUE_NULL NOPROTO { int kldsym(int fileid, int cmd, \ void *data); } 338 AUE_JAIL NOPROTO { int jail(struct jail *jail); } 339 AUE_NULL UNIMPL pioctl 340 AUE_SIGPROCMASK NOPROTO { int sigprocmask(int how, \ const sigset_t *set, sigset_t *oset); } 341 AUE_SIGSUSPEND NOPROTO { int sigsuspend(const sigset_t *sigmask); } 342 AUE_SIGACTION COMPAT4 { int freebsd32_sigaction(int sig, \ struct sigaction32 *act, \ struct sigaction32 *oact); } 343 AUE_SIGPENDING NOPROTO { int sigpending(sigset_t *set); } 344 AUE_SIGRETURN COMPAT4 { int freebsd32_sigreturn( \ const struct freebsd4_freebsd32_ucontext *sigcntxp); } ; XXX implement 345 AUE_SIGWAIT UNIMPL sigtimedwait ; XXX implement 346 AUE_NULL UNIMPL sigwaitinfo 347 AUE_NULL NOPROTO { int __acl_get_file(const char *path, \ acl_type_t type, struct acl *aclp); } 348 AUE_NULL NOPROTO { int __acl_set_file(const char *path, \ acl_type_t type, struct acl *aclp); } 349 AUE_NULL NOPROTO { int __acl_get_fd(int filedes, \ acl_type_t type, struct acl *aclp); } 350 AUE_NULL NOPROTO { int __acl_set_fd(int filedes, \ acl_type_t type, struct acl *aclp); } 351 AUE_NULL NOPROTO { int __acl_delete_file(const char *path, \ acl_type_t type); } 352 AUE_NULL NOPROTO { int __acl_delete_fd(int filedes, \ acl_type_t type); } 353 AUE_NULL NOPROTO { int __acl_aclcheck_file(const char *path, \ acl_type_t type, struct acl *aclp); } 354 AUE_NULL NOPROTO { int __acl_aclcheck_fd(int filedes, \ acl_type_t type, struct acl *aclp); } 355 AUE_EXTATTRCTL NOPROTO { int extattrctl(const char *path, int cmd, \ const char *filename, int attrnamespace, \ const char *attrname); } 356 AUE_EXTATTR_SET_FILE NOPROTO { int extattr_set_file( \ const char *path, int attrnamespace, \ const char *attrname, void *data, \ size_t nbytes); } 357 AUE_EXTATTR_GET_FILE NOPROTO { ssize_t extattr_get_file( \ const char *path, int attrnamespace, \ const char *attrname, void *data, \ size_t nbytes); } 358 AUE_EXTATTR_DELETE_FILE NOPROTO { int extattr_delete_file( \ const char *path, int attrnamespace, \ const char *attrname); } 359 AUE_NULL UNIMPL aio_waitcomplete 360 AUE_GETRESUID NOPROTO { int getresuid(uid_t *ruid, uid_t *euid, \ uid_t *suid); } 361 AUE_GETRESGID NOPROTO { int getresgid(gid_t *rgid, gid_t *egid, \ gid_t *sgid); } 362 AUE_KQUEUE NOPROTO { int kqueue(void); } 363 AUE_NULL STD { int freebsd32_kevent(int fd, \ const struct kevent32 *changelist, \ int nchanges, \ struct kevent32 *eventlist, int nevents, \ const struct timespec32 *timeout); } 364 AUE_NULL UNIMPL __cap_get_proc 365 AUE_NULL UNIMPL __cap_set_proc 366 AUE_NULL UNIMPL __cap_get_fd 367 AUE_NULL UNIMPL __cap_get_file 368 AUE_NULL UNIMPL __cap_set_fd 369 AUE_NULL UNIMPL __cap_set_file 370 AUE_NULL UNIMPL lkmressys 371 AUE_EXTATTR_SET_FD NOPROTO { int extattr_set_fd(int fd, \ int attrnamespace, const char *attrname, \ void *data, size_t nbytes); } 372 AUE_EXTATTR_GET_FD NOPROTO { ssize_t extattr_get_fd(int fd, \ int attrnamespace, const char *attrname, \ void *data, size_t nbytes); } 373 AUE_EXTATTR_DELETE_FD NOPROTO { int extattr_delete_fd(int fd, \ int attrnamespace, \ const char *attrname); } 374 AUE_NULL NOPROTO { int __setugid(int flag); } 375 AUE_NULL UNIMPL nfsclnt 376 AUE_EACCESS NOPROTO { int eaccess(char *path, int flags); } 377 AUE_NULL UNIMPL afs_syscall 378 AUE_NMOUNT NOPROTO { int nmount(struct iovec *iovp, \ unsigned int iovcnt, int flags); } 379 AUE_NULL NOPROTO { int kse_exit(void); } 380 AUE_NULL NOPROTO { int kse_wakeup(struct kse_mailbox *mbx); } 381 AUE_NULL NOPROTO { int kse_create(struct kse_mailbox *mbx, \ int newgroup); } 382 AUE_NULL NOPROTO { int kse_thr_interrupt( \ struct kse_thr_mailbox *tmbx); } 383 AUE_NULL NOPROTO { int kse_release(void); } 384 AUE_NULL UNIMPL __mac_get_proc 385 AUE_NULL UNIMPL __mac_set_proc 386 AUE_NULL UNIMPL __mac_get_fd 387 AUE_NULL UNIMPL __mac_get_file 388 AUE_NULL UNIMPL __mac_set_fd 389 AUE_NULL UNIMPL __mac_set_file 390 AUE_NULL NOPROTO { int kenv(int what, const char *name, \ char *value, int len); } 391 AUE_LCHFLAGS NOPROTO { int lchflags(const char *path, int flags); } 392 AUE_NULL NOPROTO { int uuidgen(struct uuid *store, \ int count); } 393 AUE_SENDFILE STD { int freebsd32_sendfile(int fd, int s, \ u_int32_t offsetlo, u_int32_t offsethi, \ size_t nbytes, struct sf_hdtr32 *hdtr, \ off_t *sbytes, int flags); } 394 AUE_NULL UNIMPL mac_syscall 395 AUE_GETFSSTAT NOPROTO { int getfsstat(struct statfs *buf, \ long bufsize, int flags); } 396 AUE_STATFS NOPROTO { int statfs(char *path, \ struct statfs *buf); } 397 AUE_FSTATFS NOPROTO { int fstatfs(int fd, struct statfs *buf); } 398 AUE_FHSTATFS NOPROTO { int fhstatfs(const struct fhandle *u_fhp, \ struct statfs *buf); } 399 AUE_NULL UNIMPL nosys ; XXX implement these? 400 AUE_NULL UNIMPL ksem_close 401 AUE_NULL UNIMPL ksem_post 402 AUE_NULL UNIMPL ksem_wait 403 AUE_NULL UNIMPL ksem_trywait 404 AUE_NULL UNIMPL ksem_init 405 AUE_NULL UNIMPL ksem_open 406 AUE_NULL UNIMPL ksem_unlink 407 AUE_NULL UNIMPL ksem_getvalue 408 AUE_NULL UNIMPL ksem_destroy 409 AUE_NULL UNIMPL __mac_get_pid 410 AUE_NULL UNIMPL __mac_get_link 411 AUE_NULL UNIMPL __mac_set_link 412 AUE_EXTATTR_SET_LINK UNIMPL extattr_set_link 413 AUE_EXTATTR_GET_LINK UNIMPL extattr_get_link 414 AUE_EXTATTR_DELETE_LINK UNIMPL extattr_delete_link 415 AUE_NULL UNIMPL __mac_execve 416 AUE_SIGACTION STD { int freebsd32_sigaction(int sig, \ struct sigaction32 *act, \ struct sigaction32 *oact); } 417 AUE_SIGRETURN STD { int freebsd32_sigreturn( \ const struct freebsd32_ucontext *sigcntxp); } 418 AUE_NULL UNIMPL __xstat 419 AUE_NULL UNIMPL __xfstat 420 AUE_NULL UNIMPL __xlstat 421 AUE_NULL STD { int freebsd32_getcontext( \ struct freebsd32_ucontext *ucp); } 422 AUE_NULL STD { int freebsd32_setcontext( \ const struct freebsd32_ucontext *ucp); } 423 AUE_NULL STD { int freebsd32_swapcontext( \ struct freebsd32_ucontext *oucp, \ const struct freebsd32_ucontext *ucp); } 424 AUE_SWAPOFF UNIMPL swapoff 425 AUE_NULL UNIMPL __acl_get_link 426 AUE_NULL UNIMPL __acl_set_link 427 AUE_NULL UNIMPL __acl_delete_link 428 AUE_NULL UNIMPL __acl_aclcheck_link ; XXX implement 429 AUE_SIGWAIT UNIMPL sigwait 430 AUE_NULL NOPROTO { int thr_create(ucontext_t *ctx, long *id, \ int flag s); } 431 AUE_NULL NOPROTO { void thr_exit(long *state); } 432 AUE_NULL NOPROTO { int thr_self(long *id); } 433 AUE_NULL NOPROTO { int thr_kill(long id, int sig); } 434 AUE_NULL NOPROTO { int _umtx_lock(struct umtx *umtx); } 435 AUE_NULL NOPROTO { int _umtx_unlock(struct umtx *umtx); } 436 AUE_NULL NOPROTO { int jail_attach(int jid); } 437 AUE_EXTATTR_LIST_FD UNIMPL extattr_list_fd 438 AUE_EXTATTR_LIST_FILE UNIMPL extattr_list_file 439 AUE_EXTATTR_LIST_LINK UNIMPL extattr_list_link 440 AUE_NULL UNIMPL kse_switchin 441 AUE_NULL UNIMPL ksem_timedwait 442 AUE_NULL NOPROTO { int thr_suspend( \ const struct timespec *timeout); } 443 AUE_NULL NOPROTO { int thr_wake(long id); } 444 AUE_MODUNLOAD NOPROTO { int kldunloadf(int fileid, int flags); } 445 AUE_AUDIT UNIMPL audit 446 AUE_AUDITON UNIMPL auditon 447 AUE_GETAUID UNIMPL getauid 448 AUE_SETAUID UNIMPL setauid 449 AUE_GETAUDIT UNIMPL getaudit 450 AUE_SETAUDIT UNIMPL setaudit 451 AUE_GETAUDIT_ADDR UNIMPL getaudit_addr 452 AUE_SETAUDIT_ADDR UNIMPL setaudit_addr 453 AUE_AUDITCTL UNIMPL auditctl -454 AUE_NULL UNIMPL _umtx_op +454 AUE_NULL STD { int freebsd32_umtx_op(void *obj, int op,\ + uintptr_t val, void *uaddr, \ + void *uaddr2); } 455 AUE_NULL UNIMPL thr_new 456 AUE_NULL NOPROTO { int sigqueue(pid_t pid, int signum, \ void *value); } 457 AUE_NULL UNIMPL kmq_open 458 AUE_NULL UNIMPL kmq_setattr 459 AUE_NULL UNIMPL kmq_timedreceive 460 AUE_NULL UNIMPL kmq_timedsend 461 AUE_NULL UNIMPL kmq_notify 462 AUE_NULL UNIMPL kmq_unlink 463 AUE_NULL NOPROTO { int abort2(const char *why, int nargs, void **args); } 464 AUE_NULL NOPROTO { int thr_set_name(long id, const char *name); } 465 AUE_NULL UNIMPL aio_fsync 466 AUE_RTPRIO NOPROTO { int rtprio_thread(int function, \ lwpid_t lwpid, struct rtprio *rtp); } 467 AUE_NULL UNIMPL nosys 468 AUE_NULL UNIMPL nosys 469 AUE_NULL UNIMPL __getpath_fromfd 470 AUE_NULL UNIMPL __getpath_fromaddr Index: head/sys/kern/kern_umtx.c =================================================================== --- head/sys/kern/kern_umtx.c (revision 162535) +++ head/sys/kern/kern_umtx.c (revision 162536) @@ -1,2126 +1,2484 @@ /*- * Copyright (c) 2004, David Xu * Copyright (c) 2002, Jeffrey Roberson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); +#include "opt_compat.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include +#ifdef COMPAT_IA32 +#include +#endif + #define TYPE_SIMPLE_LOCK 0 #define TYPE_SIMPLE_WAIT 1 #define TYPE_NORMAL_UMUTEX 2 #define TYPE_PI_UMUTEX 3 #define TYPE_PP_UMUTEX 4 #define TYPE_CV 5 /* Key to represent a unique userland synchronous object */ struct umtx_key { int hash; int type; int shared; union { struct { vm_object_t object; uintptr_t offset; } shared; struct { struct vmspace *vs; uintptr_t addr; } private; struct { void *a; uintptr_t b; } both; } info; }; /* Priority inheritance mutex info. */ struct umtx_pi { /* Owner thread */ struct thread *pi_owner; /* Reference count */ int pi_refcount; /* List entry to link umtx holding by thread */ TAILQ_ENTRY(umtx_pi) pi_link; /* List entry in hash */ TAILQ_ENTRY(umtx_pi) pi_hashlink; /* List for waiters */ TAILQ_HEAD(,umtx_q) pi_blocked; /* Identify a userland lock object */ struct umtx_key pi_key; }; /* A userland synchronous object user. */ struct umtx_q { /* Linked list for the hash. */ TAILQ_ENTRY(umtx_q) uq_link; /* Umtx key. */ struct umtx_key uq_key; /* Umtx flags. */ int uq_flags; #define UQF_UMTXQ 0x0001 /* The thread waits on. */ struct thread *uq_thread; /* * Blocked on PI mutex. read can use chain lock * or sched_lock, write must have both chain lock and * sched_lock being hold. */ struct umtx_pi *uq_pi_blocked; /* On blocked list */ TAILQ_ENTRY(umtx_q) uq_lockq; /* Thread contending with us */ TAILQ_HEAD(,umtx_pi) uq_pi_contested; /* Inherited priority from PP mutex */ u_char uq_inherited_pri; }; TAILQ_HEAD(umtxq_head, umtx_q); /* Userland lock object's wait-queue chain */ struct umtxq_chain { /* Lock for this chain. */ struct mtx uc_lock; /* List of sleep queues. */ struct umtxq_head uc_queue; /* Busy flag */ char uc_busy; /* Chain lock waiters */ int uc_waiters; /* All PI in the list */ TAILQ_HEAD(,umtx_pi) uc_pi_list; }; #define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED) /* * Don't propagate time-sharing priority, there is a security reason, * a user can simply introduce PI-mutex, let thread A lock the mutex, * and let another thread B block on the mutex, because B is * sleeping, its priority will be boosted, this causes A's priority to * be boosted via priority propagating too and will never be lowered even * if it is using 100%CPU, this is unfair to other processes. */ #define UPRI(td) (((td)->td_ksegrp->kg_user_pri >= PRI_MIN_TIMESHARE &&\ (td)->td_ksegrp->kg_user_pri <= PRI_MAX_TIMESHARE) ?\ PRI_MAX_TIMESHARE : (td)->td_ksegrp->kg_user_pri) #define GOLDEN_RATIO_PRIME 2654404609U #define UMTX_CHAINS 128 #define UMTX_SHIFTS (__WORD_BIT - 7) #define THREAD_SHARE 0 #define PROCESS_SHARE 1 #define AUTO_SHARE 2 #define GET_SHARE(flags) \ (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE) static uma_zone_t umtx_pi_zone; static struct umtxq_chain umtxq_chains[UMTX_CHAINS]; static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory"); static int umtx_pi_allocated; SYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW, 0, "umtx debug"); SYSCTL_INT(_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD, &umtx_pi_allocated, 0, "Allocated umtx_pi"); static void umtxq_sysinit(void *); static void umtxq_hash(struct umtx_key *key); static struct umtxq_chain *umtxq_getchain(struct umtx_key *key); static void umtxq_lock(struct umtx_key *key); static void umtxq_unlock(struct umtx_key *key); static void umtxq_busy(struct umtx_key *key); static void umtxq_unbusy(struct umtx_key *key); static void umtxq_insert(struct umtx_q *uq); static void umtxq_remove(struct umtx_q *uq); static int umtxq_sleep(struct umtx_q *uq, const char *wmesg, int timo); static int umtxq_count(struct umtx_key *key); static int umtxq_signal(struct umtx_key *key, int nr_wakeup); static int umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2); static int umtx_key_get(void *addr, int type, int share, struct umtx_key *key); static void umtx_key_release(struct umtx_key *key); static struct umtx_pi *umtx_pi_alloc(void); static void umtx_pi_free(struct umtx_pi *pi); static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags); static void umtx_thread_cleanup(struct thread *td); static void umtx_exec_hook(void *arg __unused, struct proc *p __unused, struct image_params *imgp __unused); SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL); static void umtxq_sysinit(void *arg __unused) { int i; umtx_pi_zone = uma_zcreate("umtx pi", sizeof(struct umtx_pi), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); for (i = 0; i < UMTX_CHAINS; ++i) { mtx_init(&umtxq_chains[i].uc_lock, "umtxql", NULL, MTX_DEF | MTX_DUPOK); TAILQ_INIT(&umtxq_chains[i].uc_queue); TAILQ_INIT(&umtxq_chains[i].uc_pi_list); umtxq_chains[i].uc_busy = 0; umtxq_chains[i].uc_waiters = 0; } EVENTHANDLER_REGISTER(process_exec, umtx_exec_hook, NULL, EVENTHANDLER_PRI_ANY); } struct umtx_q * umtxq_alloc(void) { struct umtx_q *uq; uq = malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK | M_ZERO); TAILQ_INIT(&uq->uq_pi_contested); uq->uq_inherited_pri = PRI_MAX; return (uq); } void umtxq_free(struct umtx_q *uq) { free(uq, M_UMTX); } static inline void umtxq_hash(struct umtx_key *key) { unsigned n = (uintptr_t)key->info.both.a + key->info.both.b; key->hash = ((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS; } static inline int umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2) { return (k1->type == k2->type && k1->info.both.a == k2->info.both.a && k1->info.both.b == k2->info.both.b); } static inline struct umtxq_chain * umtxq_getchain(struct umtx_key *key) { return (&umtxq_chains[key->hash]); } /* * Set chain to busy state when following operation * may be blocked (kernel mutex can not be used). */ static inline void umtxq_busy(struct umtx_key *key) { struct umtxq_chain *uc; uc = umtxq_getchain(key); mtx_assert(&uc->uc_lock, MA_OWNED); while (uc->uc_busy != 0) { uc->uc_waiters++; msleep(uc, &uc->uc_lock, 0, "umtxqb", 0); uc->uc_waiters--; } uc->uc_busy = 1; } /* * Unbusy a chain. */ static inline void umtxq_unbusy(struct umtx_key *key) { struct umtxq_chain *uc; uc = umtxq_getchain(key); mtx_assert(&uc->uc_lock, MA_OWNED); KASSERT(uc->uc_busy != 0, ("not busy")); uc->uc_busy = 0; if (uc->uc_waiters) wakeup_one(uc); } /* * Lock a chain. */ static inline void umtxq_lock(struct umtx_key *key) { struct umtxq_chain *uc; uc = umtxq_getchain(key); mtx_lock(&uc->uc_lock); } /* * Unlock a chain. */ static inline void umtxq_unlock(struct umtx_key *key) { struct umtxq_chain *uc; uc = umtxq_getchain(key); mtx_unlock(&uc->uc_lock); } /* * Insert a thread onto the umtx queue. */ static inline void umtxq_insert(struct umtx_q *uq) { struct umtxq_chain *uc; uc = umtxq_getchain(&uq->uq_key); UMTXQ_LOCKED_ASSERT(uc); TAILQ_INSERT_TAIL(&uc->uc_queue, uq, uq_link); uq->uq_flags |= UQF_UMTXQ; } /* * Remove thread from the umtx queue. */ static inline void umtxq_remove(struct umtx_q *uq) { struct umtxq_chain *uc; uc = umtxq_getchain(&uq->uq_key); UMTXQ_LOCKED_ASSERT(uc); if (uq->uq_flags & UQF_UMTXQ) { TAILQ_REMOVE(&uc->uc_queue, uq, uq_link); uq->uq_flags &= ~UQF_UMTXQ; } } /* * Check if there are multiple waiters */ static int umtxq_count(struct umtx_key *key) { struct umtxq_chain *uc; struct umtx_q *uq; int count = 0; uc = umtxq_getchain(key); UMTXQ_LOCKED_ASSERT(uc); TAILQ_FOREACH(uq, &uc->uc_queue, uq_link) { if (umtx_key_match(&uq->uq_key, key)) { if (++count > 1) break; } } return (count); } /* * Check if there are multiple PI waiters and returns first * waiter. */ static int umtxq_count_pi(struct umtx_key *key, struct umtx_q **first) { struct umtxq_chain *uc; struct umtx_q *uq; int count = 0; *first = NULL; uc = umtxq_getchain(key); UMTXQ_LOCKED_ASSERT(uc); TAILQ_FOREACH(uq, &uc->uc_queue, uq_link) { if (umtx_key_match(&uq->uq_key, key)) { if (++count > 1) break; *first = uq; } } return (count); } /* * Wake up threads waiting on an userland object. */ static int umtxq_signal(struct umtx_key *key, int n_wake) { struct umtxq_chain *uc; struct umtx_q *uq, *next; int ret; ret = 0; uc = umtxq_getchain(key); UMTXQ_LOCKED_ASSERT(uc); TAILQ_FOREACH_SAFE(uq, &uc->uc_queue, uq_link, next) { if (umtx_key_match(&uq->uq_key, key)) { umtxq_remove(uq); wakeup(uq); if (++ret >= n_wake) break; } } return (ret); } /* * Wake up specified thread. */ static inline void umtxq_signal_thread(struct umtx_q *uq) { struct umtxq_chain *uc; uc = umtxq_getchain(&uq->uq_key); UMTXQ_LOCKED_ASSERT(uc); umtxq_remove(uq); wakeup(uq); } /* * Put thread into sleep state, before sleeping, check if * thread was removed from umtx queue. */ static inline int umtxq_sleep(struct umtx_q *uq, const char *wmesg, int timo) { struct umtxq_chain *uc; int error; uc = umtxq_getchain(&uq->uq_key); UMTXQ_LOCKED_ASSERT(uc); if (!(uq->uq_flags & UQF_UMTXQ)) return (0); error = msleep(uq, &uc->uc_lock, PCATCH, wmesg, timo); if (error == EWOULDBLOCK) error = ETIMEDOUT; return (error); } /* * Convert userspace address into unique logical address. */ static int umtx_key_get(void *addr, int type, int share, struct umtx_key *key) { struct thread *td = curthread; vm_map_t map; vm_map_entry_t entry; vm_pindex_t pindex; vm_prot_t prot; boolean_t wired; key->type = type; if (share == THREAD_SHARE) { key->shared = 0; key->info.private.vs = td->td_proc->p_vmspace; key->info.private.addr = (uintptr_t)addr; } else if (share == PROCESS_SHARE || share == AUTO_SHARE) { map = &td->td_proc->p_vmspace->vm_map; if (vm_map_lookup(&map, (vm_offset_t)addr, VM_PROT_WRITE, &entry, &key->info.shared.object, &pindex, &prot, &wired) != KERN_SUCCESS) { return EFAULT; } if ((share == PROCESS_SHARE) || (share == AUTO_SHARE && VM_INHERIT_SHARE == entry->inheritance)) { key->shared = 1; key->info.shared.offset = entry->offset + entry->start - (vm_offset_t)addr; vm_object_reference(key->info.shared.object); } else { key->shared = 0; key->info.private.vs = td->td_proc->p_vmspace; key->info.private.addr = (uintptr_t)addr; } vm_map_lookup_done(map, entry); } umtxq_hash(key); return (0); } /* * Release key. */ static inline void umtx_key_release(struct umtx_key *key) { if (key->shared) vm_object_deallocate(key->info.shared.object); } /* * Lock a umtx object. */ static int -_do_lock(struct thread *td, struct umtx *umtx, uintptr_t id, int timo) +_do_lock_umtx(struct thread *td, struct umtx *umtx, uintptr_t id, int timo) { struct umtx_q *uq; intptr_t owner; intptr_t old; int error = 0; uq = td->td_umtxq; /* * Care must be exercised when dealing with umtx structure. It * can fault on any access. */ for (;;) { /* * Try the uncontested case. This should be done in userland. */ owner = casuptr((intptr_t *)&umtx->u_owner, UMTX_UNOWNED, id); /* The acquire succeeded. */ if (owner == UMTX_UNOWNED) return (0); /* The address was invalid. */ if (owner == -1) return (EFAULT); /* If no one owns it but it is contested try to acquire it. */ if (owner == UMTX_CONTESTED) { owner = casuptr((intptr_t *)&umtx->u_owner, UMTX_CONTESTED, id | UMTX_CONTESTED); if (owner == UMTX_CONTESTED) return (0); /* The address was invalid. */ if (owner == -1) return (EFAULT); /* If this failed the lock has changed, restart. */ continue; } /* * If we caught a signal, we have retried and now * exit immediately. */ if (error != 0) return (error); if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK, AUTO_SHARE, &uq->uq_key)) != 0) return (error); umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); umtxq_insert(uq); umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); /* * Set the contested bit so that a release in user space * knows to use the system call for unlock. If this fails * either some one else has acquired the lock or it has been * released. */ old = casuptr((intptr_t *)&umtx->u_owner, owner, owner | UMTX_CONTESTED); /* The address was invalid. */ if (old == -1) { umtxq_lock(&uq->uq_key); umtxq_remove(uq); umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); return (EFAULT); } /* * We set the contested bit, sleep. Otherwise the lock changed * and we need to retry or we lost a race to the thread * unlocking the umtx. */ umtxq_lock(&uq->uq_key); if (old == owner) error = umtxq_sleep(uq, "umtx", timo); umtxq_remove(uq); umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); } return (0); } /* * Lock a umtx object. */ static int -do_lock(struct thread *td, struct umtx *umtx, uintptr_t id, +do_lock_umtx(struct thread *td, struct umtx *umtx, uintptr_t id, struct timespec *timeout) { struct timespec ts, ts2, ts3; struct timeval tv; int error; if (timeout == NULL) { - error = _do_lock(td, umtx, id, 0); + error = _do_lock_umtx(td, umtx, id, 0); /* Mutex locking is restarted if it is interrupted. */ if (error == EINTR) error = ERESTART; } else { getnanouptime(&ts); timespecadd(&ts, timeout); TIMESPEC_TO_TIMEVAL(&tv, timeout); for (;;) { - error = _do_lock(td, umtx, id, tvtohz(&tv)); + error = _do_lock_umtx(td, umtx, id, tvtohz(&tv)); if (error != ETIMEDOUT) break; getnanouptime(&ts2); if (timespeccmp(&ts2, &ts, >=)) { error = ETIMEDOUT; break; } ts3 = ts; timespecsub(&ts3, &ts2); TIMESPEC_TO_TIMEVAL(&tv, &ts3); } /* Timed-locking is not restarted. */ if (error == ERESTART) error = EINTR; } return (error); } /* * Unlock a umtx object. */ static int -do_unlock(struct thread *td, struct umtx *umtx, uintptr_t id) +do_unlock_umtx(struct thread *td, struct umtx *umtx, uintptr_t id) { struct umtx_key key; intptr_t owner; intptr_t old; int error; int count; /* * Make sure we own this mtx. * * XXX Need a {fu,su}ptr this is not correct on arch where * sizeof(intptr_t) != sizeof(long). */ owner = fuword(&umtx->u_owner); if (owner == -1) return (EFAULT); if ((owner & ~UMTX_CONTESTED) != id) return (EPERM); /* This should be done in userland */ if ((owner & UMTX_CONTESTED) == 0) { old = casuptr((intptr_t *)&umtx->u_owner, owner, UMTX_UNOWNED); if (old == -1) return (EFAULT); if (old == owner) return (0); owner = old; } /* We should only ever be in here for contested locks */ if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK, AUTO_SHARE, &key)) != 0) return (error); umtxq_lock(&key); umtxq_busy(&key); count = umtxq_count(&key); umtxq_unlock(&key); /* * When unlocking the umtx, it must be marked as unowned if * there is zero or one thread only waiting for it. * Otherwise, it must be marked as contested. */ old = casuptr((intptr_t *)&umtx->u_owner, owner, count <= 1 ? UMTX_UNOWNED : UMTX_CONTESTED); umtxq_lock(&key); umtxq_signal(&key,1); umtxq_unbusy(&key); umtxq_unlock(&key); umtx_key_release(&key); if (old == -1) return (EFAULT); if (old != owner) return (EINVAL); return (0); } +#ifdef COMPAT_IA32 + /* + * Lock a umtx object. + */ +static int +_do_lock_umtx32(struct thread *td, uint32_t *m, uint32_t id, int timo) +{ + struct umtx_q *uq; + uint32_t owner; + uint32_t old; + int error = 0; + + uq = td->td_umtxq; + + /* + * Care must be exercised when dealing with umtx structure. It + * can fault on any access. + */ + for (;;) { + /* + * Try the uncontested case. This should be done in userland. + */ + owner = casuword32(m, UMUTEX_UNOWNED, id); + + /* The acquire succeeded. */ + if (owner == UMUTEX_UNOWNED) + return (0); + + /* The address was invalid. */ + if (owner == -1) + return (EFAULT); + + /* If no one owns it but it is contested try to acquire it. */ + if (owner == UMUTEX_CONTESTED) { + owner = casuword32(m, + UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); + if (owner == UMUTEX_CONTESTED) + return (0); + + /* The address was invalid. */ + if (owner == -1) + return (EFAULT); + + /* If this failed the lock has changed, restart. */ + continue; + } + + /* + * If we caught a signal, we have retried and now + * exit immediately. + */ + if (error != 0) + return (error); + + if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK, + AUTO_SHARE, &uq->uq_key)) != 0) + return (error); + + umtxq_lock(&uq->uq_key); + umtxq_busy(&uq->uq_key); + umtxq_insert(uq); + umtxq_unbusy(&uq->uq_key); + umtxq_unlock(&uq->uq_key); + + /* + * Set the contested bit so that a release in user space + * knows to use the system call for unlock. If this fails + * either some one else has acquired the lock or it has been + * released. + */ + old = casuword32(m, owner, owner | UMUTEX_CONTESTED); + + /* The address was invalid. */ + if (old == -1) { + umtxq_lock(&uq->uq_key); + umtxq_remove(uq); + umtxq_unlock(&uq->uq_key); + umtx_key_release(&uq->uq_key); + return (EFAULT); + } + + /* + * We set the contested bit, sleep. Otherwise the lock changed + * and we need to retry or we lost a race to the thread + * unlocking the umtx. + */ + umtxq_lock(&uq->uq_key); + if (old == owner) + error = umtxq_sleep(uq, "umtx", timo); + umtxq_remove(uq); + umtxq_unlock(&uq->uq_key); + umtx_key_release(&uq->uq_key); + } + + return (0); +} + +/* + * Lock a umtx object. + */ +static int +do_lock_umtx32(struct thread *td, void *m, uint32_t id, + struct timespec *timeout) +{ + struct timespec ts, ts2, ts3; + struct timeval tv; + int error; + + if (timeout == NULL) { + error = _do_lock_umtx32(td, m, id, 0); + /* Mutex locking is restarted if it is interrupted. */ + if (error == EINTR) + error = ERESTART; + } else { + getnanouptime(&ts); + timespecadd(&ts, timeout); + TIMESPEC_TO_TIMEVAL(&tv, timeout); + for (;;) { + error = _do_lock_umtx32(td, m, id, tvtohz(&tv)); + if (error != ETIMEDOUT) + break; + getnanouptime(&ts2); + if (timespeccmp(&ts2, &ts, >=)) { + error = ETIMEDOUT; + break; + } + ts3 = ts; + timespecsub(&ts3, &ts2); + TIMESPEC_TO_TIMEVAL(&tv, &ts3); + } + /* Timed-locking is not restarted. */ + if (error == ERESTART) + error = EINTR; + } + return (error); +} + +/* + * Unlock a umtx object. + */ +static int +do_unlock_umtx32(struct thread *td, uint32_t *m, uint32_t id) +{ + struct umtx_key key; + uint32_t owner; + uint32_t old; + int error; + int count; + + /* + * Make sure we own this mtx. + * + * XXX Need a {fu,su}ptr this is not correct on arch where + * sizeof(intptr_t) != sizeof(long). + */ + owner = fuword32(m); + if (owner == -1) + return (EFAULT); + + if ((owner & ~UMUTEX_CONTESTED) != id) + return (EPERM); + + /* This should be done in userland */ + if ((owner & UMUTEX_CONTESTED) == 0) { + old = casuword32(m, owner, UMUTEX_UNOWNED); + if (old == -1) + return (EFAULT); + if (old == owner) + return (0); + owner = old; + } + + /* We should only ever be in here for contested locks */ + if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK, AUTO_SHARE, + &key)) != 0) + return (error); + + umtxq_lock(&key); + umtxq_busy(&key); + count = umtxq_count(&key); + umtxq_unlock(&key); + + /* + * When unlocking the umtx, it must be marked as unowned if + * there is zero or one thread only waiting for it. + * Otherwise, it must be marked as contested. + */ + old = casuword32(m, owner, + count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); + umtxq_lock(&key); + umtxq_signal(&key,1); + umtxq_unbusy(&key); + umtxq_unlock(&key); + umtx_key_release(&key); + if (old == -1) + return (EFAULT); + if (old != owner) + return (EINVAL); + return (0); +} +#endif + +/* * Fetch and compare value, sleep on the address if value is not changed. */ static int -do_wait(struct thread *td, struct umtx *umtx, uintptr_t id, struct timespec *timeout) +do_wait(struct thread *td, void *addr, uintptr_t id, + struct timespec *timeout, int compat32) { struct umtx_q *uq; struct timespec ts, ts2, ts3; struct timeval tv; uintptr_t tmp; int error = 0; uq = td->td_umtxq; - if ((error = umtx_key_get(umtx, TYPE_SIMPLE_WAIT, AUTO_SHARE, + if ((error = umtx_key_get(addr, TYPE_SIMPLE_WAIT, AUTO_SHARE, &uq->uq_key)) != 0) return (error); umtxq_lock(&uq->uq_key); umtxq_insert(uq); umtxq_unlock(&uq->uq_key); - tmp = fuword(&umtx->u_owner); + if (compat32 == 0) + tmp = fuword(addr); + else + tmp = fuword32(addr); if (tmp != id) { umtxq_lock(&uq->uq_key); umtxq_remove(uq); umtxq_unlock(&uq->uq_key); } else if (timeout == NULL) { umtxq_lock(&uq->uq_key); error = umtxq_sleep(uq, "ucond", 0); umtxq_remove(uq); umtxq_unlock(&uq->uq_key); } else { getnanouptime(&ts); timespecadd(&ts, timeout); TIMESPEC_TO_TIMEVAL(&tv, timeout); umtxq_lock(&uq->uq_key); for (;;) { error = umtxq_sleep(uq, "ucond", tvtohz(&tv)); if (!(uq->uq_flags & UQF_UMTXQ)) break; if (error != ETIMEDOUT) break; umtxq_unlock(&uq->uq_key); getnanouptime(&ts2); if (timespeccmp(&ts2, &ts, >=)) { error = ETIMEDOUT; umtxq_lock(&uq->uq_key); break; } ts3 = ts; timespecsub(&ts3, &ts2); TIMESPEC_TO_TIMEVAL(&tv, &ts3); umtxq_lock(&uq->uq_key); } umtxq_remove(uq); umtxq_unlock(&uq->uq_key); } umtx_key_release(&uq->uq_key); if (error == ERESTART) error = EINTR; return (error); } /* * Wake up threads sleeping on the specified address. */ int kern_umtx_wake(struct thread *td, void *uaddr, int n_wake) { struct umtx_key key; int ret; if ((ret = umtx_key_get(uaddr, TYPE_SIMPLE_WAIT, AUTO_SHARE, &key)) != 0) return (ret); umtxq_lock(&key); ret = umtxq_signal(&key, n_wake); umtxq_unlock(&key); umtx_key_release(&key); return (0); } /* * Lock PTHREAD_PRIO_NONE protocol POSIX mutex. */ static int _do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, int timo, int try) { struct umtx_q *uq; uint32_t owner, old, id; int error = 0; id = td->td_tid; uq = td->td_umtxq; /* * Care must be exercised when dealing with umtx structure. It * can fault on any access. */ for (;;) { /* * Try the uncontested case. This should be done in userland. */ owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id); /* The acquire succeeded. */ if (owner == UMUTEX_UNOWNED) return (0); /* The address was invalid. */ if (owner == -1) return (EFAULT); /* If no one owns it but it is contested try to acquire it. */ if (owner == UMUTEX_CONTESTED) { owner = casuword32(&m->m_owner, UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); if (owner == UMUTEX_CONTESTED) return (0); /* The address was invalid. */ if (owner == -1) return (EFAULT); /* If this failed the lock has changed, restart. */ continue; } if ((flags & UMUTEX_ERROR_CHECK) != 0 && (owner & ~UMUTEX_CONTESTED) == id) return (EDEADLK); if (try != 0) return (EBUSY); /* * If we caught a signal, we have retried and now * exit immediately. */ if (error != 0) return (error); if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), &uq->uq_key)) != 0) return (error); umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); umtxq_insert(uq); umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); /* * Set the contested bit so that a release in user space * knows to use the system call for unlock. If this fails * either some one else has acquired the lock or it has been * released. */ old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED); /* The address was invalid. */ if (old == -1) { umtxq_lock(&uq->uq_key); umtxq_remove(uq); umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); return (EFAULT); } /* * We set the contested bit, sleep. Otherwise the lock changed * and we need to retry or we lost a race to the thread * unlocking the umtx. */ umtxq_lock(&uq->uq_key); if (old == owner) error = umtxq_sleep(uq, "umtxn", timo); umtxq_remove(uq); umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); } return (0); } /* * Lock PTHREAD_PRIO_NONE protocol POSIX mutex. */ /* * Unlock PTHREAD_PRIO_NONE protocol POSIX mutex. */ static int do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags) { struct umtx_key key; uint32_t owner, old, id; int error; int count; id = td->td_tid; /* * Make sure we own this mtx. */ owner = fuword32(&m->m_owner); if (owner == -1) return (EFAULT); if ((owner & ~UMUTEX_CONTESTED) != id) return (EPERM); /* This should be done in userland */ if ((owner & UMUTEX_CONTESTED) == 0) { old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED); if (old == -1) return (EFAULT); if (old == owner) return (0); owner = old; } /* We should only ever be in here for contested locks */ if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), &key)) != 0) return (error); umtxq_lock(&key); umtxq_busy(&key); count = umtxq_count(&key); umtxq_unlock(&key); /* * When unlocking the umtx, it must be marked as unowned if * there is zero or one thread only waiting for it. * Otherwise, it must be marked as contested. */ old = casuword32(&m->m_owner, owner, count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); umtxq_lock(&key); umtxq_signal(&key,1); umtxq_unbusy(&key); umtxq_unlock(&key); umtx_key_release(&key); if (old == -1) return (EFAULT); if (old != owner) return (EINVAL); return (0); } static inline struct umtx_pi * umtx_pi_alloc(void) { struct umtx_pi *pi; pi = uma_zalloc(umtx_pi_zone, M_ZERO | M_WAITOK); TAILQ_INIT(&pi->pi_blocked); atomic_add_int(&umtx_pi_allocated, 1); return (pi); } static inline void umtx_pi_free(struct umtx_pi *pi) { uma_zfree(umtx_pi_zone, pi); atomic_add_int(&umtx_pi_allocated, -1); } /* * Adjust the thread's position on a pi_state after its priority has been * changed. */ static int umtx_pi_adjust_thread(struct umtx_pi *pi, struct thread *td) { struct umtx_q *uq, *uq1, *uq2; struct thread *td1; mtx_assert(&sched_lock, MA_OWNED); if (pi == NULL) return (0); uq = td->td_umtxq; /* * Check if the thread needs to be moved on the blocked chain. * It needs to be moved if either its priority is lower than * the previous thread or higher than the next thread. */ uq1 = TAILQ_PREV(uq, umtxq_head, uq_lockq); uq2 = TAILQ_NEXT(uq, uq_lockq); if ((uq1 != NULL && UPRI(td) < UPRI(uq1->uq_thread)) || (uq2 != NULL && UPRI(td) > UPRI(uq2->uq_thread))) { /* * Remove thread from blocked chain and determine where * it should be moved to. */ TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq); TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) { td1 = uq1->uq_thread; MPASS(td1->td_proc->p_magic == P_MAGIC); if (UPRI(td1) > UPRI(td)) break; } if (uq1 == NULL) TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq); else TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq); } return (1); } /* * Propagate priority when a thread is blocked on POSIX * PI mutex. */ static void umtx_propagate_priority(struct thread *td) { struct umtx_q *uq; struct umtx_pi *pi; int pri; mtx_assert(&sched_lock, MA_OWNED); pri = UPRI(td); uq = td->td_umtxq; pi = uq->uq_pi_blocked; if (pi == NULL) return; for (;;) { td = pi->pi_owner; if (td == NULL) return; MPASS(td->td_proc != NULL); MPASS(td->td_proc->p_magic == P_MAGIC); if (UPRI(td) <= pri) return; sched_lend_user_prio(td, pri); /* * Pick up the lock that td is blocked on. */ uq = td->td_umtxq; pi = uq->uq_pi_blocked; /* Resort td on the list if needed. */ if (!umtx_pi_adjust_thread(pi, td)) break; } } /* * Unpropagate priority for a PI mutex when a thread blocked on * it is interrupted by signal or resumed by others. */ static void umtx_unpropagate_priority(struct umtx_pi *pi) { struct umtx_q *uq, *uq_owner; struct umtx_pi *pi2; int pri; mtx_assert(&sched_lock, MA_OWNED); while (pi != NULL && pi->pi_owner != NULL) { pri = PRI_MAX; uq_owner = pi->pi_owner->td_umtxq; TAILQ_FOREACH(pi2, &uq_owner->uq_pi_contested, pi_link) { uq = TAILQ_FIRST(&pi2->pi_blocked); if (uq != NULL) { if (pri > UPRI(uq->uq_thread)) pri = UPRI(uq->uq_thread); } } if (pri > uq_owner->uq_inherited_pri) pri = uq_owner->uq_inherited_pri; sched_unlend_user_prio(pi->pi_owner, pri); pi = uq_owner->uq_pi_blocked; } } /* * Insert a PI mutex into owned list. */ static void umtx_pi_setowner(struct umtx_pi *pi, struct thread *owner) { struct umtx_q *uq_owner; uq_owner = owner->td_umtxq; mtx_assert(&sched_lock, MA_OWNED); if (pi->pi_owner != NULL) panic("pi_ower != NULL"); pi->pi_owner = owner; TAILQ_INSERT_TAIL(&uq_owner->uq_pi_contested, pi, pi_link); } /* * Claim ownership of a PI mutex. */ static int umtx_pi_claim(struct umtx_pi *pi, struct thread *owner) { struct umtx_q *uq, *uq_owner; uq_owner = owner->td_umtxq; mtx_lock_spin(&sched_lock); if (pi->pi_owner == owner) { mtx_unlock_spin(&sched_lock); return (0); } if (pi->pi_owner != NULL) { /* * userland may have already messed the mutex, sigh. */ mtx_unlock_spin(&sched_lock); return (EPERM); } umtx_pi_setowner(pi, owner); uq = TAILQ_FIRST(&pi->pi_blocked); if (uq != NULL) { int pri; pri = UPRI(uq->uq_thread); if (pri < UPRI(owner)) sched_lend_user_prio(owner, pri); } mtx_unlock_spin(&sched_lock); return (0); } /* * Adjust a thread's order position in its blocked PI mutex, * this may result new priority propagating process. */ void umtx_pi_adjust(struct thread *td, u_char oldpri) { struct umtx_q *uq; struct umtx_pi *pi; uq = td->td_umtxq; mtx_assert(&sched_lock, MA_OWNED); MPASS(TD_ON_UPILOCK(td)); /* * Pick up the lock that td is blocked on. */ pi = uq->uq_pi_blocked; MPASS(pi != NULL); /* Resort the turnstile on the list. */ if (!umtx_pi_adjust_thread(pi, td)) return; /* * If our priority was lowered and we are at the head of the * turnstile, then propagate our new priority up the chain. */ if (uq == TAILQ_FIRST(&pi->pi_blocked) && UPRI(td) < oldpri) umtx_propagate_priority(td); } /* * Sleep on a PI mutex. */ static int umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, uint32_t owner, const char *wmesg, int timo) { struct umtxq_chain *uc; struct thread *td, *td1; struct umtx_q *uq1; int pri; int error = 0; td = uq->uq_thread; KASSERT(td == curthread, ("inconsistent uq_thread")); uc = umtxq_getchain(&uq->uq_key); UMTXQ_LOCKED_ASSERT(uc); umtxq_insert(uq); if (pi->pi_owner == NULL) { /* XXX * Current, We only support process private PI-mutex, * non-contended PI-mutexes are locked in userland. * Process shared PI-mutex should always be initialized * by kernel and be registered in kernel, locking should * always be done by kernel to avoid security problems. * For process private PI-mutex, we can find owner * thread and boost its priority safely. */ PROC_LOCK(curproc); td1 = thread_find(curproc, owner); mtx_lock_spin(&sched_lock); if (td1 != NULL && pi->pi_owner == NULL) { uq1 = td1->td_umtxq; umtx_pi_setowner(pi, td1); } PROC_UNLOCK(curproc); } else { mtx_lock_spin(&sched_lock); } TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) { pri = UPRI(uq1->uq_thread); if (pri > UPRI(td)) break; } if (uq1 != NULL) TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq); else TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq); uq->uq_pi_blocked = pi; td->td_flags |= TDF_UPIBLOCKED; mtx_unlock_spin(&sched_lock); umtxq_unlock(&uq->uq_key); mtx_lock_spin(&sched_lock); umtx_propagate_priority(td); mtx_unlock_spin(&sched_lock); umtxq_lock(&uq->uq_key); if (uq->uq_flags & UQF_UMTXQ) { error = msleep(uq, &uc->uc_lock, PCATCH, wmesg, timo); if (error == EWOULDBLOCK) error = ETIMEDOUT; if (uq->uq_flags & UQF_UMTXQ) { umtxq_busy(&uq->uq_key); umtxq_remove(uq); umtxq_unbusy(&uq->uq_key); } } umtxq_unlock(&uq->uq_key); mtx_lock_spin(&sched_lock); uq->uq_pi_blocked = NULL; td->td_flags &= ~TDF_UPIBLOCKED; TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq); umtx_unpropagate_priority(pi); mtx_unlock_spin(&sched_lock); umtxq_lock(&uq->uq_key); return (error); } /* * Add reference count for a PI mutex. */ static void umtx_pi_ref(struct umtx_pi *pi) { struct umtxq_chain *uc; uc = umtxq_getchain(&pi->pi_key); UMTXQ_LOCKED_ASSERT(uc); pi->pi_refcount++; } /* * Decrease reference count for a PI mutex, if the counter * is decreased to zero, its memory space is freed. */ static void umtx_pi_unref(struct umtx_pi *pi) { struct umtxq_chain *uc; int free = 0; uc = umtxq_getchain(&pi->pi_key); UMTXQ_LOCKED_ASSERT(uc); KASSERT(pi->pi_refcount > 0, ("invalid reference count")); if (--pi->pi_refcount == 0) { mtx_lock_spin(&sched_lock); if (pi->pi_owner != NULL) { TAILQ_REMOVE(&pi->pi_owner->td_umtxq->uq_pi_contested, pi, pi_link); pi->pi_owner = NULL; } KASSERT(TAILQ_EMPTY(&pi->pi_blocked), ("blocked queue not empty")); mtx_unlock_spin(&sched_lock); TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink); free = 1; } if (free) umtx_pi_free(pi); } /* * Find a PI mutex in hash table. */ static struct umtx_pi * umtx_pi_lookup(struct umtx_key *key) { struct umtxq_chain *uc; struct umtx_pi *pi; uc = umtxq_getchain(key); UMTXQ_LOCKED_ASSERT(uc); TAILQ_FOREACH(pi, &uc->uc_pi_list, pi_hashlink) { if (umtx_key_match(&pi->pi_key, key)) { return (pi); } } return (NULL); } /* * Insert a PI mutex into hash table. */ static inline void umtx_pi_insert(struct umtx_pi *pi) { struct umtxq_chain *uc; uc = umtxq_getchain(&pi->pi_key); UMTXQ_LOCKED_ASSERT(uc); TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink); } /* * Lock a PI mutex. */ static int _do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, int timo, int try) { struct umtx_q *uq; struct umtx_pi *pi, *new_pi; uint32_t id, owner, old; int error; id = td->td_tid; uq = td->td_umtxq; if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags), &uq->uq_key)) != 0) return (error); for (;;) { pi = NULL; umtxq_lock(&uq->uq_key); pi = umtx_pi_lookup(&uq->uq_key); if (pi == NULL) { umtxq_unlock(&uq->uq_key); new_pi = umtx_pi_alloc(); new_pi->pi_key = uq->uq_key; umtxq_lock(&uq->uq_key); pi = umtx_pi_lookup(&uq->uq_key); if (pi != NULL) umtx_pi_free(new_pi); else { umtx_pi_insert(new_pi); pi = new_pi; } } umtx_pi_ref(pi); umtxq_unlock(&uq->uq_key); /* * Care must be exercised when dealing with umtx structure. It * can fault on any access. */ /* * Try the uncontested case. This should be done in userland. */ owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id); /* The acquire succeeded. */ if (owner == UMUTEX_UNOWNED) { error = 0; break; } /* The address was invalid. */ if (owner == -1) { error = EFAULT; break; } /* If no one owns it but it is contested try to acquire it. */ if (owner == UMUTEX_CONTESTED) { owner = casuword32(&m->m_owner, UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); if (owner == UMUTEX_CONTESTED) { umtxq_lock(&uq->uq_key); error = umtx_pi_claim(pi, td); umtxq_unlock(&uq->uq_key); break; } /* The address was invalid. */ if (owner == -1) { error = EFAULT; break; } /* If this failed the lock has changed, restart. */ umtxq_lock(&uq->uq_key); umtx_pi_unref(pi); umtxq_unlock(&uq->uq_key); pi = NULL; continue; } if ((flags & UMUTEX_ERROR_CHECK) != 0 && (owner & ~UMUTEX_CONTESTED) == id) { error = EDEADLK; break; } if (try != 0) { error = EBUSY; break; } /* * If we caught a signal, we have retried and now * exit immediately. */ if (error != 0) break; umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); umtxq_unlock(&uq->uq_key); /* * Set the contested bit so that a release in user space * knows to use the system call for unlock. If this fails * either some one else has acquired the lock or it has been * released. */ old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED); /* The address was invalid. */ if (old == -1) { umtxq_lock(&uq->uq_key); umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); error = EFAULT; break; } umtxq_lock(&uq->uq_key); umtxq_unbusy(&uq->uq_key); /* * We set the contested bit, sleep. Otherwise the lock changed * and we need to retry or we lost a race to the thread * unlocking the umtx. */ if (old == owner) error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED, "umtxpi", timo); umtx_pi_unref(pi); umtxq_unlock(&uq->uq_key); pi = NULL; } if (pi != NULL) { umtxq_lock(&uq->uq_key); umtx_pi_unref(pi); umtxq_unlock(&uq->uq_key); } umtx_key_release(&uq->uq_key); return (error); } /* * Unlock a PI mutex. */ static int do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags) { struct umtx_key key; struct umtx_q *uq_first, *uq_first2, *uq_me; struct umtx_pi *pi, *pi2; uint32_t owner, old, id; int error; int count; int pri; id = td->td_tid; /* * Make sure we own this mtx. */ owner = fuword32(&m->m_owner); if (owner == -1) return (EFAULT); if ((owner & ~UMUTEX_CONTESTED) != id) return (EPERM); /* This should be done in userland */ if ((owner & UMUTEX_CONTESTED) == 0) { old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED); if (old == -1) return (EFAULT); if (old == owner) return (0); owner = old; } /* We should only ever be in here for contested locks */ if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags), &key)) != 0) return (error); umtxq_lock(&key); umtxq_busy(&key); count = umtxq_count_pi(&key, &uq_first); if (uq_first != NULL) { pi = uq_first->uq_pi_blocked; if (pi->pi_owner != curthread) { umtxq_unbusy(&key); umtxq_unlock(&key); /* userland messed the mutex */ return (EPERM); } uq_me = curthread->td_umtxq; mtx_lock_spin(&sched_lock); pi->pi_owner = NULL; TAILQ_REMOVE(&uq_me->uq_pi_contested, pi, pi_link); uq_first = TAILQ_FIRST(&pi->pi_blocked); pri = PRI_MAX; TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) { uq_first2 = TAILQ_FIRST(&pi2->pi_blocked); if (uq_first2 != NULL) { if (pri > UPRI(uq_first2->uq_thread)) pri = UPRI(uq_first2->uq_thread); } } sched_unlend_user_prio(curthread, pri); mtx_unlock_spin(&sched_lock); } umtxq_unlock(&key); /* * When unlocking the umtx, it must be marked as unowned if * there is zero or one thread only waiting for it. * Otherwise, it must be marked as contested. */ old = casuword32(&m->m_owner, owner, count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); umtxq_lock(&key); if (uq_first != NULL) umtxq_signal_thread(uq_first); umtxq_unbusy(&key); umtxq_unlock(&key); umtx_key_release(&key); if (old == -1) return (EFAULT); if (old != owner) return (EINVAL); return (0); } /* * Lock a PP mutex. */ static int _do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, int timo, int try) { struct umtx_q *uq, *uq2; struct umtx_pi *pi; uint32_t ceiling; uint32_t owner, id; int error, pri, old_inherited_pri, su; id = td->td_tid; uq = td->td_umtxq; if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags), &uq->uq_key)) != 0) return (error); su = (suser(td) == 0); for (;;) { old_inherited_pri = uq->uq_inherited_pri; umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); umtxq_unlock(&uq->uq_key); ceiling = RTP_PRIO_MAX - fuword32(&m->m_ceilings[0]); if (ceiling > RTP_PRIO_MAX) { error = EINVAL; goto out; } mtx_lock_spin(&sched_lock); if (UPRI(td) < PRI_MIN_REALTIME + ceiling) { mtx_unlock_spin(&sched_lock); error = EINVAL; goto out; } if (su && PRI_MIN_REALTIME + ceiling < uq->uq_inherited_pri) { uq->uq_inherited_pri = PRI_MIN_REALTIME + ceiling; if (uq->uq_inherited_pri < UPRI(td)) sched_lend_user_prio(td, uq->uq_inherited_pri); } mtx_unlock_spin(&sched_lock); owner = casuword32(&m->m_owner, UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); if (owner == UMUTEX_CONTESTED) { error = 0; break; } /* The address was invalid. */ if (owner == -1) { error = EFAULT; break; } if ((flags & UMUTEX_ERROR_CHECK) != 0 && (owner & ~UMUTEX_CONTESTED) == id) { error = EDEADLK; break; } if (try != 0) { error = EBUSY; break; } /* * If we caught a signal, we have retried and now * exit immediately. */ if (error != 0) break; umtxq_lock(&uq->uq_key); umtxq_insert(uq); umtxq_unbusy(&uq->uq_key); error = umtxq_sleep(uq, "umtxpp", timo); umtxq_remove(uq); umtxq_unlock(&uq->uq_key); mtx_lock_spin(&sched_lock); uq->uq_inherited_pri = old_inherited_pri; pri = PRI_MAX; TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { uq2 = TAILQ_FIRST(&pi->pi_blocked); if (uq2 != NULL) { if (pri > UPRI(uq2->uq_thread)) pri = UPRI(uq2->uq_thread); } } if (pri > uq->uq_inherited_pri) pri = uq->uq_inherited_pri; sched_unlend_user_prio(td, pri); mtx_unlock_spin(&sched_lock); } if (error != 0) { mtx_lock_spin(&sched_lock); uq->uq_inherited_pri = old_inherited_pri; pri = PRI_MAX; TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { uq2 = TAILQ_FIRST(&pi->pi_blocked); if (uq2 != NULL) { if (pri > UPRI(uq2->uq_thread)) pri = UPRI(uq2->uq_thread); } } if (pri > uq->uq_inherited_pri) pri = uq->uq_inherited_pri; sched_unlend_user_prio(td, pri); mtx_unlock_spin(&sched_lock); } out: umtxq_lock(&uq->uq_key); umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); return (error); } /* * Unlock a PP mutex. */ static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags) { struct umtx_key key; struct umtx_q *uq, *uq2; struct umtx_pi *pi; uint32_t owner, id; uint32_t rceiling; int error, pri, new_inherited_pri, su; id = td->td_tid; uq = td->td_umtxq; su = (suser(td) == 0); /* * Make sure we own this mtx. */ owner = fuword32(&m->m_owner); if (owner == -1) return (EFAULT); if ((owner & ~UMUTEX_CONTESTED) != id) return (EPERM); error = copyin(&m->m_ceilings[1], &rceiling, sizeof(uint32_t)); if (error != 0) return (error); if (rceiling == -1) new_inherited_pri = PRI_MAX; else { rceiling = RTP_PRIO_MAX - rceiling; if (rceiling > RTP_PRIO_MAX) return (EINVAL); new_inherited_pri = PRI_MIN_REALTIME + rceiling; } if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags), &key)) != 0) return (error); umtxq_lock(&key); umtxq_busy(&key); umtxq_unlock(&key); /* * For priority protected mutex, always set unlocked state * to UMUTEX_CONTESTED, so that userland always enters kernel * to lock the mutex, it is necessary because thread priority * has to be adjusted for such mutex. */ error = suword32(&m->m_owner, UMUTEX_CONTESTED); umtxq_lock(&key); if (error == 0) umtxq_signal(&key, 1); umtxq_unbusy(&key); umtxq_unlock(&key); if (error == -1) error = EFAULT; else { mtx_lock_spin(&sched_lock); if (su != 0) uq->uq_inherited_pri = new_inherited_pri; pri = PRI_MAX; TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { uq2 = TAILQ_FIRST(&pi->pi_blocked); if (uq2 != NULL) { if (pri > UPRI(uq2->uq_thread)) pri = UPRI(uq2->uq_thread); } } if (pri > uq->uq_inherited_pri) pri = uq->uq_inherited_pri; sched_unlend_user_prio(td, pri); mtx_unlock_spin(&sched_lock); } umtx_key_release(&key); return (error); } static int do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling, uint32_t *old_ceiling) { struct umtx_q *uq; uint32_t save_ceiling; uint32_t owner, id; uint32_t flags; int error; flags = fuword32(&m->m_flags); if ((flags & UMUTEX_PRIO_PROTECT) == 0) return (EINVAL); if (ceiling > RTP_PRIO_MAX) return (EINVAL); id = td->td_tid; uq = td->td_umtxq; if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags), &uq->uq_key)) != 0) return (error); for (;;) { umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); umtxq_unlock(&uq->uq_key); save_ceiling = fuword32(&m->m_ceilings[0]); owner = casuword32(&m->m_owner, UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); if (owner == UMUTEX_CONTESTED) { suword32(&m->m_ceilings[0], ceiling); suword32(&m->m_owner, UMUTEX_CONTESTED); error = 0; break; } /* The address was invalid. */ if (owner == -1) { error = EFAULT; break; } if ((owner & ~UMUTEX_CONTESTED) == id) { suword32(&m->m_ceilings[0], ceiling); error = 0; break; } /* * If we caught a signal, we have retried and now * exit immediately. */ if (error != 0) break; /* * We set the contested bit, sleep. Otherwise the lock changed * and we need to retry or we lost a race to the thread * unlocking the umtx. */ umtxq_lock(&uq->uq_key); umtxq_insert(uq); umtxq_unbusy(&uq->uq_key); error = umtxq_sleep(uq, "umtxpp", 0); umtxq_remove(uq); umtxq_unlock(&uq->uq_key); } umtxq_lock(&uq->uq_key); if (error == 0) umtxq_signal(&uq->uq_key, INT_MAX); umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); if (error == 0 && old_ceiling != NULL) suword32(old_ceiling, save_ceiling); return (error); } static int _do_lock_umutex(struct thread *td, struct umutex *m, int flags, int timo, int try) { switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { case 0: return (_do_lock_normal(td, m, flags, timo, try)); case UMUTEX_PRIO_INHERIT: return (_do_lock_pi(td, m, flags, timo, try)); case UMUTEX_PRIO_PROTECT: return (_do_lock_pp(td, m, flags, timo, try)); } return (EINVAL); } /* * Lock a userland POSIX mutex. */ static int do_lock_umutex(struct thread *td, struct umutex *m, struct timespec *timeout, int try) { struct timespec ts, ts2, ts3; struct timeval tv; uint32_t flags; int error; flags = fuword32(&m->m_flags); if (flags == -1) return (EFAULT); if (timeout == NULL) { error = _do_lock_umutex(td, m, flags, 0, try); /* Mutex locking is restarted if it is interrupted. */ if (error == EINTR) error = ERESTART; } else { getnanouptime(&ts); timespecadd(&ts, timeout); TIMESPEC_TO_TIMEVAL(&tv, timeout); for (;;) { error = _do_lock_umutex(td, m, flags, tvtohz(&tv), try); if (error != ETIMEDOUT) break; getnanouptime(&ts2); if (timespeccmp(&ts2, &ts, >=)) { error = ETIMEDOUT; break; } ts3 = ts; timespecsub(&ts3, &ts2); TIMESPEC_TO_TIMEVAL(&tv, &ts3); } /* Timed-locking is not restarted. */ if (error == ERESTART) error = EINTR; } return (error); } /* * Unlock a userland POSIX mutex. */ static int do_unlock_umutex(struct thread *td, struct umutex *m) { uint32_t flags; flags = fuword32(&m->m_flags); if (flags == -1) return (EFAULT); switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { case 0: return (do_unlock_normal(td, m, flags)); case UMUTEX_PRIO_INHERIT: return (do_unlock_pi(td, m, flags)); case UMUTEX_PRIO_PROTECT: return (do_unlock_pp(td, m, flags)); } return (EINVAL); } int _umtx_lock(struct thread *td, struct _umtx_lock_args *uap) /* struct umtx *umtx */ { - return _do_lock(td, uap->umtx, td->td_tid, 0); + return _do_lock_umtx(td, uap->umtx, td->td_tid, 0); } int _umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap) /* struct umtx *umtx */ { - return do_unlock(td, uap->umtx, td->td_tid); + return do_unlock_umtx(td, uap->umtx, td->td_tid); } -int -_umtx_op(struct thread *td, struct _umtx_op_args *uap) +static int +__umtx_op_lock_umtx(struct thread *td, struct _umtx_op_args *uap) { - struct timespec timeout; - struct timespec *ts; + struct timespec *ts, timeout; int error; - switch(uap->op) { - case UMTX_OP_MUTEX_LOCK: - /* Allow a null timespec (wait forever). */ - if (uap->uaddr2 == NULL) - ts = NULL; - else { - error = copyin(uap->uaddr2, &timeout, sizeof(timeout)); - if (error != 0) - break; - if (timeout.tv_nsec >= 1000000000 || - timeout.tv_nsec < 0) { - error = EINVAL; - break; - } - ts = &timeout; + /* Allow a null timespec (wait forever). */ + if (uap->uaddr2 == NULL) + ts = NULL; + else { + error = copyin(uap->uaddr2, &timeout, sizeof(timeout)); + if (error != 0) + return (error); + if (timeout.tv_nsec >= 1000000000 || + timeout.tv_nsec < 0) { + return (EINVAL); } - error = do_lock_umutex(td, uap->obj, ts, 0); - break; - case UMTX_OP_MUTEX_UNLOCK: - error = do_unlock_umutex(td, uap->obj); - break; - case UMTX_OP_LOCK: - /* Allow a null timespec (wait forever). */ - if (uap->uaddr2 == NULL) - ts = NULL; - else { - error = copyin(uap->uaddr2, &timeout, sizeof(timeout)); - if (error != 0) - break; - if (timeout.tv_nsec >= 1000000000 || - timeout.tv_nsec < 0) { - error = EINVAL; - break; - } - ts = &timeout; + ts = &timeout; + } + return (do_lock_umtx(td, uap->obj, uap->val, ts)); +} + +static int +__umtx_op_unlock_umtx(struct thread *td, struct _umtx_op_args *uap) +{ + return (do_unlock_umtx(td, uap->obj, uap->val)); +} + +static int +__umtx_op_wait(struct thread *td, struct _umtx_op_args *uap) +{ + struct timespec *ts, timeout; + int error; + + if (uap->uaddr2 == NULL) + ts = NULL; + else { + error = copyin(uap->uaddr2, &timeout, sizeof(timeout)); + if (error != 0) + return (error); + if (timeout.tv_nsec >= 1000000000 || + timeout.tv_nsec < 0) + return (EINVAL); + ts = &timeout; + } + return do_wait(td, uap->obj, uap->val, ts, 0); +} + +static int +__umtx_op_wake(struct thread *td, struct _umtx_op_args *uap) +{ + return (kern_umtx_wake(td, uap->obj, uap->val)); +} + +static int +__umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap) +{ + struct timespec *ts, timeout; + int error; + + /* Allow a null timespec (wait forever). */ + if (uap->uaddr2 == NULL) + ts = NULL; + else { + error = copyin(uap->uaddr2, &timeout, + sizeof(timeout)); + if (error != 0) + return (error); + if (timeout.tv_nsec >= 1000000000 || + timeout.tv_nsec < 0) { + return (EINVAL); } - error = do_lock(td, uap->obj, uap->val, ts); - break; - case UMTX_OP_UNLOCK: - error = do_unlock(td, uap->obj, uap->val); - break; - case UMTX_OP_WAIT: - /* Allow a null timespec (wait forever). */ - if (uap->uaddr2 == NULL) - ts = NULL; - else { - error = copyin(uap->uaddr2, &timeout, sizeof(timeout)); - if (error != 0) - break; - if (timeout.tv_nsec >= 1000000000 || - timeout.tv_nsec < 0) { - error = EINVAL; - break; - } - ts = &timeout; - } - error = do_wait(td, uap->obj, uap->val, ts); - break; - case UMTX_OP_WAKE: - error = kern_umtx_wake(td, uap->obj, uap->val); - break; - case UMTX_OP_MUTEX_TRYLOCK: - error = do_lock_umutex(td, uap->obj, NULL, 1); - break; - case UMTX_OP_SET_CEILING: - error = do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1); - break; - default: - error = EINVAL; - break; + ts = &timeout; } + return do_lock_umutex(td, uap->obj, ts, 0); +} + +static int +__umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap) +{ + return do_lock_umutex(td, uap->obj, NULL, 1); +} + +static int +__umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap) +{ + return do_unlock_umutex(td, uap->obj); +} + +static int +__umtx_op_set_ceiling(struct thread *td, struct _umtx_op_args *uap) +{ + return do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1); +} + +typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap); + +static _umtx_op_func op_table[] = { + __umtx_op_lock_umtx, /* UMTX_OP_LOCK */ + __umtx_op_unlock_umtx, /* UMTX_OP_UNLOCK */ + __umtx_op_wait, /* UMTX_OP_WAIT */ + __umtx_op_wake, /* UMTX_OP_WAKE */ + __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_TRYLOCK */ + __umtx_op_lock_umutex, /* UMTX_OP_MUTEX_LOCK */ + __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */ + __umtx_op_set_ceiling /* UMTX_OP_SET_CEILING */ +}; + +int +_umtx_op(struct thread *td, struct _umtx_op_args *uap) +{ + if (uap->op >= 0 && uap->op < UMTX_OP_MAX) + return (*op_table[uap->op])(td, uap); + return (EINVAL); +} + +#ifdef COMPAT_IA32 + +struct timespec32 { + u_int32_t tv_sec; + u_int32_t tv_nsec; +}; + +static inline int +copyin_timeout32(void *addr, struct timespec *tsp) +{ + struct timespec32 ts32; + int error; + + error = copyin(addr, &ts32, sizeof(struct timespec32)); + if (error == 0) { + tsp->tv_sec = ts32.tv_sec; + tsp->tv_nsec = ts32.tv_nsec; + } return (error); } + +static int +__umtx_op_lock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap) +{ + struct timespec *ts, timeout; + int error; + + /* Allow a null timespec (wait forever). */ + if (uap->uaddr2 == NULL) + ts = NULL; + else { + error = copyin_timeout32(uap->uaddr2, &timeout); + if (error != 0) + return (error); + if (timeout.tv_nsec >= 1000000000 || + timeout.tv_nsec < 0) { + return (EINVAL); + } + ts = &timeout; + } + return (do_lock_umtx32(td, uap->obj, uap->val, ts)); +} + +static int +__umtx_op_unlock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap) +{ + return (do_unlock_umtx32(td, uap->obj, (uint32_t)uap->val)); +} + +static int +__umtx_op_wait_compat32(struct thread *td, struct _umtx_op_args *uap) +{ + struct timespec *ts, timeout; + int error; + + if (uap->uaddr2 == NULL) + ts = NULL; + else { + error = copyin_timeout32(uap->uaddr2, &timeout); + if (error != 0) + return (error); + if (timeout.tv_nsec >= 1000000000 || + timeout.tv_nsec < 0) + return (EINVAL); + ts = &timeout; + } + return do_wait(td, uap->obj, uap->val, ts, 1); +} + +static int +__umtx_op_lock_umutex_compat32(struct thread *td, struct _umtx_op_args *uap) +{ + struct timespec *ts, timeout; + int error; + + /* Allow a null timespec (wait forever). */ + if (uap->uaddr2 == NULL) + ts = NULL; + else { + error = copyin_timeout32(uap->uaddr2, &timeout); + if (error != 0) + return (error); + if (timeout.tv_nsec >= 1000000000 || + timeout.tv_nsec < 0) + return (EINVAL); + ts = &timeout; + } + return do_lock_umutex(td, uap->obj, ts, 0); +} + +static _umtx_op_func op_table_compat32[] = { + __umtx_op_lock_umtx_compat32, /* UMTX_OP_LOCK */ + __umtx_op_unlock_umtx_compat32, /* UMTX_OP_UNLOCK */ + __umtx_op_wait_compat32, /* UMTX_OP_WAIT */ + __umtx_op_wake, /* UMTX_OP_WAKE */ + __umtx_op_lock_umutex_compat32, /* UMTX_OP_MUTEX_TRYLOCK */ + __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_LOCK */ + __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */ + __umtx_op_set_ceiling /* UMTX_OP_SET_CEILING */ +}; + +int +freebsd32_umtx_op(struct thread *td, struct freebsd32_umtx_op_args *uap) +{ + if (uap->op >= 0 && uap->op < UMTX_OP_MAX) + return (*op_table_compat32[uap->op])(td, + (struct _umtx_op_args *)uap); + return (EINVAL); +} +#endif void umtx_thread_init(struct thread *td) { td->td_umtxq = umtxq_alloc(); td->td_umtxq->uq_thread = td; } void umtx_thread_fini(struct thread *td) { umtxq_free(td->td_umtxq); } /* * It will be called when new thread is created, e.g fork(). */ void umtx_thread_alloc(struct thread *td) { struct umtx_q *uq; uq = td->td_umtxq; uq->uq_inherited_pri = PRI_MAX; KASSERT(uq->uq_flags == 0, ("uq_flags != 0")); KASSERT(uq->uq_thread == td, ("uq_thread != td")); KASSERT(uq->uq_pi_blocked == NULL, ("uq_pi_blocked != NULL")); KASSERT(TAILQ_EMPTY(&uq->uq_pi_contested), ("uq_pi_contested is not empty")); } /* * exec() hook. */ static void umtx_exec_hook(void *arg __unused, struct proc *p __unused, struct image_params *imgp __unused) { umtx_thread_cleanup(curthread); } /* * thread_exit() hook. */ void umtx_thread_exit(struct thread *td) { umtx_thread_cleanup(td); } /* * clean up umtx data. */ static void umtx_thread_cleanup(struct thread *td) { struct umtx_q *uq; struct umtx_pi *pi; if ((uq = td->td_umtxq) == NULL) return; mtx_lock_spin(&sched_lock); uq->uq_inherited_pri = PRI_MAX; while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) { pi->pi_owner = NULL; TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link); } td->td_flags &= ~TDF_UBORROWING; mtx_unlock_spin(&sched_lock); } Index: head/sys/sys/umtx.h =================================================================== --- head/sys/sys/umtx.h (revision 162535) +++ head/sys/sys/umtx.h (revision 162536) @@ -1,169 +1,170 @@ /*- * Copyright (c) 2002, Jeffrey Roberson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ * */ #ifndef _SYS_UMTX_H_ #define _SYS_UMTX_H_ #include #include /* * See pthread_* */ #define UMTX_UNOWNED 0x0 #define UMTX_CONTESTED LONG_MIN struct umtx { uintptr_t u_owner; /* Owner of the mutex. */ }; #define USYNC_PROCESS_SHARED 0x0001 /* Process shared sync objs */ #define UMUTEX_UNOWNED 0x0 #define UMUTEX_CONTESTED 0x80000000 #define UMUTEX_ERROR_CHECK 0x0002 /* Error-checking mutex */ #define UMUTEX_PRIO_INHERIT 0x0004 /* Priority inherited mutex */ #define UMUTEX_PRIO_PROTECT 0x0008 /* Priority protect mutex */ struct umutex { __lwpid_t m_owner; /* Owner of the mutex */ uint32_t m_flags; /* Flags of the mutex */ uint32_t m_ceilings[2]; /* Priority protect ceiling */ uint32_t m_spare[4]; /* Spare space */ }; /* op code for _umtx_op */ #define UMTX_OP_LOCK 0 #define UMTX_OP_UNLOCK 1 #define UMTX_OP_WAIT 2 #define UMTX_OP_WAKE 3 #define UMTX_OP_MUTEX_TRYLOCK 4 #define UMTX_OP_MUTEX_LOCK 5 #define UMTX_OP_MUTEX_UNLOCK 6 #define UMTX_OP_SET_CEILING 7 +#define UMTX_OP_MAX 8 #ifndef _KERNEL /* * System call for userland mutex operations. * Bug: assume sizeof(uintptr_t) == sizeof(long) */ int _umtx_op(void *obj, int op, uintptr_t val, void *uaddr, void *uaddr2); /* * Old (deprecated) userland mutex system calls. */ int _umtx_lock(struct umtx *mtx); int _umtx_unlock(struct umtx *mtx); /* * Standard api. Try uncontested acquire/release and asks the * kernel to resolve failures. */ static __inline void umtx_init(struct umtx *umtx) { umtx->u_owner = UMTX_UNOWNED; } static __inline uintptr_t umtx_owner(struct umtx *umtx) { return (umtx->u_owner & ~LONG_MIN); } static __inline int umtx_lock(struct umtx *umtx, uintptr_t id) { if (atomic_cmpset_acq_ptr(&umtx->u_owner, UMTX_UNOWNED, id) == 0) if (_umtx_lock(umtx) == -1) return (errno); return (0); } static __inline int umtx_trylock(struct umtx *umtx, uintptr_t id) { if (atomic_cmpset_acq_ptr(&umtx->u_owner, UMTX_UNOWNED, id) == 0) return (EBUSY); return (0); } static __inline int umtx_timedlock(struct umtx *umtx, uintptr_t id, const struct timespec *timeout) { if (atomic_cmpset_acq_ptr(&umtx->u_owner, UMTX_UNOWNED, id) == 0) if (_umtx_op(umtx, UMTX_OP_LOCK, id, 0, __DECONST(void *, timeout)) == -1) return (errno); return (0); } static __inline int umtx_unlock(struct umtx *umtx, uintptr_t id) { if (atomic_cmpset_rel_ptr(&umtx->u_owner, id, UMTX_UNOWNED) == 0) if (_umtx_unlock(umtx) == -1) return (errno); return (0); } static __inline int umtx_wait(long *p, long val, const struct timespec *timeout) { if (_umtx_op(p, UMTX_OP_WAIT, val, 0, __DECONST(void *, timeout)) == -1) return (errno); return (0); } /* Wake threads waiting on a user address. */ static __inline int umtx_wake(long *p, int nr_wakeup) { if (_umtx_op(p, UMTX_OP_WAKE, nr_wakeup, 0, 0) == -1) return (errno); return (0); } #else struct thread; struct umtx_q *umtxq_alloc(void); void umtxq_free(struct umtx_q *); int kern_umtx_wake(struct thread *td, void *uaddr, int n_wake); void umtx_pi_adjust(struct thread *td, u_char oldpri); void umtx_thread_init(struct thread *td); void umtx_thread_fini(struct thread *td); void umtx_thread_alloc(struct thread *td); void umtx_thread_exit(struct thread *td); #endif /* !_KERNEL */ #endif /* !_SYS_UMTX_H_ */