diff --git a/sys/powerpc/include/counter.h b/sys/powerpc/include/counter.h index 68e292cde7f6..90e6400ad562 100644 --- a/sys/powerpc/include/counter.h +++ b/sys/powerpc/include/counter.h @@ -1,162 +1,164 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012, 2013 Konstantin Belousov * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef __MACHINE_COUNTER_H__ #define __MACHINE_COUNTER_H__ #include #ifdef INVARIANTS #include #endif #define EARLY_COUNTER &__pcpu[0].pc_early_dummy_counter #ifdef __powerpc64__ #define counter_enter() do {} while (0) #define counter_exit() do {} while (0) #ifdef IN_SUBR_COUNTER_C static inline uint64_t counter_u64_read_one(uint64_t *p, int cpu) { return (*(uint64_t *)((char *)p + UMA_PCPU_ALLOC_SIZE * cpu)); } static inline uint64_t counter_u64_fetch_inline(uint64_t *p) { uint64_t r; int i; r = 0; CPU_FOREACH(i) r += counter_u64_read_one((uint64_t *)p, i); return (r); } static void counter_u64_zero_one_cpu(void *arg) { *((uint64_t *)((char *)arg + UMA_PCPU_ALLOC_SIZE * PCPU_GET(cpuid))) = 0; } static inline void counter_u64_zero_inline(counter_u64_t c) { smp_rendezvous(smp_no_rendezvous_barrier, counter_u64_zero_one_cpu, smp_no_rendezvous_barrier, c); } #endif #define counter_u64_add_protected(c, i) counter_u64_add(c, i) static inline void counter_u64_add(counter_u64_t c, int64_t inc) { uint64_t ccpu, old; __asm __volatile("\n" "1:\n\t" "mfsprg %0, 0\n\t" "ldarx %1, %0, %2\n\t" "add %1, %1, %3\n\t" "stdcx. %1, %0, %2\n\t" "bne- 1b" : "=&b" (ccpu), "=&r" (old) : "r" ((char *)c - (char *)&__pcpu[0]), "r" (inc) : "cr0", "memory"); } #else /* !64bit */ +#include + #define counter_enter() critical_enter() #define counter_exit() critical_exit() #ifdef IN_SUBR_COUNTER_C /* XXXKIB non-atomic 64bit read */ static inline uint64_t counter_u64_read_one(uint64_t *p, int cpu) { return (*(uint64_t *)((char *)p + UMA_PCPU_ALLOC_SIZE * cpu)); } static inline uint64_t counter_u64_fetch_inline(uint64_t *p) { uint64_t r; int i; r = 0; for (i = 0; i < mp_ncpus; i++) r += counter_u64_read_one((uint64_t *)p, i); return (r); } /* XXXKIB non-atomic 64bit store, might interrupt increment */ static void counter_u64_zero_one_cpu(void *arg) { *((uint64_t *)((char *)arg + UMA_PCPU_ALLOC_SIZE * PCPU_GET(cpuid))) = 0; } static inline void counter_u64_zero_inline(counter_u64_t c) { smp_rendezvous(smp_no_rendezvous_barrier, counter_u64_zero_one_cpu, smp_no_rendezvous_barrier, c); } #endif #define counter_u64_add_protected(c, inc) do { \ CRITICAL_ASSERT(curthread); \ *(uint64_t *)zpcpu_get(c) += (inc); \ } while (0) static inline void counter_u64_add(counter_u64_t c, int64_t inc) { counter_enter(); counter_u64_add_protected(c, inc); counter_exit(); } #endif /* 64bit */ #endif /* ! __MACHINE_COUNTER_H__ */ diff --git a/sys/sys/_lock.h b/sys/sys/_lock.h index ef167b9ab5ec..7057463fbde2 100644 --- a/sys/sys/_lock.h +++ b/sys/sys/_lock.h @@ -1,75 +1,75 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Berkeley Software Design Inc's name may not be used to endorse or * promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _SYS__LOCK_H_ #define _SYS__LOCK_H_ struct lock_object { const char *lo_name; /* Individual lock name. */ - u_int lo_flags; - u_int lo_data; /* General class specific data. */ + unsigned int lo_flags; + unsigned int lo_data; /* General class specific data. */ struct witness *lo_witness; /* Data for witness. */ }; #ifdef _KERNEL /* * If any of WITNESS, INVARIANTS, or KTR_LOCK KTR tracing has been enabled, * then turn on LOCK_DEBUG. When this option is on, extra debugging * facilities such as tracking the file and line number of lock operations * are enabled. Also, mutex locking operations are not inlined to avoid * bloat from all the extra debugging code. We also have to turn on all the * calling conventions for this debugging code in modules so that modules can * work with both debug and non-debug kernels. */ #if (defined(KLD_MODULE) && !defined(KLD_TIED)) || defined(WITNESS) || defined(INVARIANTS) || \ defined(LOCK_PROFILING) || defined(KTR) #define LOCK_DEBUG 1 #else #define LOCK_DEBUG 0 #endif /* * In the LOCK_DEBUG case, use the filename and line numbers for debugging * operations. Otherwise, use default values to avoid the unneeded bloat. */ #if LOCK_DEBUG > 0 #define LOCK_FILE_LINE_ARG_DEF , const char *file, int line #define LOCK_FILE_LINE_ARG , file, line #define LOCK_FILE __FILE__ #define LOCK_LINE __LINE__ #else #define LOCK_FILE_LINE_ARG_DEF #define LOCK_FILE_LINE_ARG #define LOCK_FILE NULL #define LOCK_LINE 0 #endif #endif /* _KERNEL */ #endif /* !_SYS__LOCK_H_ */ diff --git a/sys/sys/_mutex.h b/sys/sys/_mutex.h index 0fcd5a2c2809..7241f9f1d31c 100644 --- a/sys/sys/_mutex.h +++ b/sys/sys/_mutex.h @@ -1,64 +1,66 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Berkeley Software Design Inc's name may not be used to endorse or * promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _SYS__MUTEX_H_ #define _SYS__MUTEX_H_ +#include +#include #include /* * Sleep/spin mutex. * * All mutex implementations must always have a member called mtx_lock. * Other locking primitive structures are not allowed to use this name * for their members. * If this rule needs to change, the bits in the mutex implementation must * be modified appropriately. */ struct mtx { struct lock_object lock_object; /* Common lock properties. */ - volatile uintptr_t mtx_lock; /* Owner and flags. */ + volatile __uintptr_t mtx_lock; /* Owner and flags. */ }; /* * Members of struct mtx_padalign must mirror members of struct mtx. * mtx_padalign mutexes can use the mtx(9) API transparently without * modification. * Pad-aligned mutexes used within structures should generally be the * first member of the struct. Otherwise, the compiler can generate * additional padding for the struct to keep a correct alignment for * the mutex. */ struct mtx_padalign { struct lock_object lock_object; /* Common lock properties. */ - volatile uintptr_t mtx_lock; /* Owner and flags. */ + volatile __uintptr_t mtx_lock; /* Owner and flags. */ } __aligned(CACHE_LINE_SIZE); #endif /* !_SYS__MUTEX_H_ */ diff --git a/sys/sys/_rmlock.h b/sys/sys/_rmlock.h index 38375b623a65..9c8893efe19c 100644 --- a/sys/sys/_rmlock.h +++ b/sys/sys/_rmlock.h @@ -1,82 +1,88 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 2007 Stephan Uphoff * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the author nor the names of any co-contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _SYS__RMLOCK_H_ #define _SYS__RMLOCK_H_ +#include +#include +#include +#include +#include + +struct thread; + /* * Mostly reader/occasional writer lock. */ LIST_HEAD(rmpriolist,rm_priotracker); struct rm_queue { struct rm_queue *volatile rmq_next; struct rm_queue *volatile rmq_prev; }; struct rmlock { struct lock_object lock_object; volatile cpuset_t rm_writecpus; LIST_HEAD(,rm_priotracker) rm_activeReaders; union { struct lock_object _rm_wlock_object; struct mtx _rm_lock_mtx; struct sx _rm_lock_sx; } _rm_lock; }; #define rm_wlock_object _rm_lock._rm_wlock_object #define rm_lock_mtx _rm_lock._rm_lock_mtx #define rm_lock_sx _rm_lock._rm_lock_sx struct rm_priotracker { struct rm_queue rmp_cpuQueue; /* Must be first */ struct rmlock *rmp_rmlock; struct thread *rmp_thread; int rmp_flags; LIST_ENTRY(rm_priotracker) rmp_qentry; }; -#include - struct rmslock_pcpu; struct rmslock { struct mtx mtx; struct thread *owner; struct rmslock_pcpu *pcpu; int writers; int readers; int debug_readers; }; #endif /* !_SYS__RMLOCK_H_ */ diff --git a/sys/sys/_rwlock.h b/sys/sys/_rwlock.h index d360cfe2a4e9..8c15771cb172 100644 --- a/sys/sys/_rwlock.h +++ b/sys/sys/_rwlock.h @@ -1,61 +1,63 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2006 John Baldwin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _SYS__RWLOCK_H_ #define _SYS__RWLOCK_H_ +#include +#include #include /* * Reader/writer lock. * * All reader/writer lock implementations must always have a member * called rw_lock. Other locking primitive structures are not allowed to * use this name for their members. * If this rule needs to change, the bits in the reader/writer lock * implementation must be modified appropriately. */ struct rwlock { struct lock_object lock_object; - volatile uintptr_t rw_lock; + volatile __uintptr_t rw_lock; }; /* * Members of struct rwlock_padalign must mirror members of struct rwlock. * rwlock_padalign rwlocks can use the rwlock(9) API transparently without * modification. * Pad-aligned rwlocks used within structures should generally be the * first member of the struct. Otherwise, the compiler can generate * additional padding for the struct to keep a correct alignment for * the rwlock. */ struct rwlock_padalign { struct lock_object lock_object; - volatile uintptr_t rw_lock; + volatile __uintptr_t rw_lock; } __aligned(CACHE_LINE_SIZE); #endif /* !_SYS__RWLOCK_H_ */ diff --git a/sys/sys/_sx.h b/sys/sys/_sx.h index f70462c9a008..c6fd485620c6 100644 --- a/sys/sys/_sx.h +++ b/sys/sys/_sx.h @@ -1,42 +1,45 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2007 Attilio Rao * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice(s), this list of conditions and the following disclaimer as * the first lines of this file unmodified other than the possible * addition of one or more copyright notices. * 2. Redistributions in binary form must reproduce the above copyright * notice(s), this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH * DAMAGE. */ #ifndef _SYS__SX_H_ #define _SYS__SX_H_ +#include +#include + /* * Shared/exclusive lock main structure definition. */ struct sx { struct lock_object lock_object; - volatile uintptr_t sx_lock; + volatile __uintptr_t sx_lock; }; #endif /* !_SYS__SX_H_ */ diff --git a/sys/sys/file.h b/sys/sys/file.h index 820ad1015573..a2dc02e523d8 100644 --- a/sys/sys/file.h +++ b/sys/sys/file.h @@ -1,508 +1,511 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 1982, 1986, 1989, 1993 * The Regents of the University of California. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _SYS_FILE_H_ #define _SYS_FILE_H_ #ifndef _KERNEL #include /* XXX */ #include #include #else -#include -#include +#include #include #include +#include +#include +#include #include struct filedesc; +struct proc; struct stat; struct thread; struct uio; struct knote; struct vnode; struct nameidata; #endif /* _KERNEL */ #define DTYPE_NONE 0 /* not yet initialized */ #define DTYPE_VNODE 1 /* file */ #define DTYPE_SOCKET 2 /* communications endpoint */ #define DTYPE_PIPE 3 /* pipe */ #define DTYPE_FIFO 4 /* fifo (named pipe) */ #define DTYPE_KQUEUE 5 /* event queue */ #define DTYPE_CRYPTO 6 /* crypto */ #define DTYPE_MQUEUE 7 /* posix message queue */ #define DTYPE_SHM 8 /* swap-backed shared memory */ #define DTYPE_SEM 9 /* posix semaphore */ #define DTYPE_PTS 10 /* pseudo teletype master device */ #define DTYPE_DEV 11 /* Device specific fd type */ #define DTYPE_PROCDESC 12 /* process descriptor */ #define DTYPE_EVENTFD 13 /* eventfd */ #define DTYPE_TIMERFD 14 /* timerfd */ #ifdef _KERNEL struct file; struct filecaps; struct kaiocb; struct kinfo_file; struct ucred; #define FOF_OFFSET 0x01 /* Use the offset in uio argument */ #define FOF_NOLOCK 0x02 /* Do not take FOFFSET_LOCK */ #define FOF_NEXTOFF_R 0x04 /* Also update f_nextoff[UIO_READ] */ #define FOF_NEXTOFF_W 0x08 /* Also update f_nextoff[UIO_WRITE] */ #define FOF_NOUPDATE 0x10 /* Do not update f_offset */ off_t foffset_lock(struct file *fp, int flags); void foffset_lock_uio(struct file *fp, struct uio *uio, int flags); void foffset_unlock(struct file *fp, off_t val, int flags); void foffset_unlock_uio(struct file *fp, struct uio *uio, int flags); static inline off_t foffset_get(struct file *fp) { return (foffset_lock(fp, FOF_NOLOCK)); } typedef int fo_rdwr_t(struct file *fp, struct uio *uio, struct ucred *active_cred, int flags, struct thread *td); typedef int fo_truncate_t(struct file *fp, off_t length, struct ucred *active_cred, struct thread *td); typedef int fo_ioctl_t(struct file *fp, u_long com, void *data, struct ucred *active_cred, struct thread *td); typedef int fo_poll_t(struct file *fp, int events, struct ucred *active_cred, struct thread *td); typedef int fo_kqfilter_t(struct file *fp, struct knote *kn); typedef int fo_stat_t(struct file *fp, struct stat *sb, struct ucred *active_cred); typedef int fo_close_t(struct file *fp, struct thread *td); typedef int fo_chmod_t(struct file *fp, mode_t mode, struct ucred *active_cred, struct thread *td); typedef int fo_chown_t(struct file *fp, uid_t uid, gid_t gid, struct ucred *active_cred, struct thread *td); typedef int fo_sendfile_t(struct file *fp, int sockfd, struct uio *hdr_uio, struct uio *trl_uio, off_t offset, size_t nbytes, off_t *sent, int flags, struct thread *td); typedef int fo_seek_t(struct file *fp, off_t offset, int whence, struct thread *td); typedef int fo_fill_kinfo_t(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp); typedef int fo_mmap_t(struct file *fp, vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot, vm_prot_t cap_maxprot, int flags, vm_ooffset_t foff, struct thread *td); typedef int fo_aio_queue_t(struct file *fp, struct kaiocb *job); typedef int fo_add_seals_t(struct file *fp, int flags); typedef int fo_get_seals_t(struct file *fp, int *flags); typedef int fo_fallocate_t(struct file *fp, off_t offset, off_t len, struct thread *td); typedef int fo_fspacectl_t(struct file *fp, int cmd, off_t *offset, off_t *length, int flags, struct ucred *active_cred, struct thread *td); typedef int fo_cmp_t(struct file *fp, struct file *fp1, struct thread *td); typedef int fo_spare_t(struct file *fp); typedef int fo_flags_t; struct fileops { fo_rdwr_t *fo_read; fo_rdwr_t *fo_write; fo_truncate_t *fo_truncate; fo_ioctl_t *fo_ioctl; fo_poll_t *fo_poll; fo_kqfilter_t *fo_kqfilter; fo_stat_t *fo_stat; fo_close_t *fo_close; fo_chmod_t *fo_chmod; fo_chown_t *fo_chown; fo_sendfile_t *fo_sendfile; fo_seek_t *fo_seek; fo_fill_kinfo_t *fo_fill_kinfo; fo_mmap_t *fo_mmap; fo_aio_queue_t *fo_aio_queue; fo_add_seals_t *fo_add_seals; fo_get_seals_t *fo_get_seals; fo_fallocate_t *fo_fallocate; fo_fspacectl_t *fo_fspacectl; fo_cmp_t *fo_cmp; fo_spare_t *fo_spares[8]; /* Spare slots */ fo_flags_t fo_flags; /* DFLAG_* below */ }; #define DFLAG_PASSABLE 0x01 /* may be passed via unix sockets. */ #define DFLAG_SEEKABLE 0x02 /* seekable / nonsequential */ #endif /* _KERNEL */ #if defined(_KERNEL) || defined(_WANT_FILE) /* * Kernel descriptor table. * One entry for each open kernel vnode and socket. * * Below is the list of locks that protects members in struct file. * * (a) f_vnode lock required (shared allows both reads and writes) * (f) updated with atomics and blocking on sleepq * (d) cdevpriv_mtx * none not locked */ #if __BSD_VISIBLE struct fadvise_info { int fa_advice; /* (f) FADV_* type. */ off_t fa_start; /* (f) Region start. */ off_t fa_end; /* (f) Region end. */ }; struct file { volatile u_int f_flag; /* see fcntl.h */ volatile u_int f_count; /* reference count */ void *f_data; /* file descriptor specific data */ const struct fileops *f_ops; /* File operations */ struct vnode *f_vnode; /* NULL or applicable vnode */ struct ucred *f_cred; /* associated credentials. */ short f_type; /* descriptor type */ short f_vnread_flags; /* (f) Sleep lock for f_offset */ /* * DTYPE_VNODE specific fields. */ union { int16_t f_seqcount[2]; /* (a) Count of seq. reads and writes. */ int f_pipegen; }; off_t f_nextoff[2]; /* next expected read/write offset. */ union { struct cdev_privdata *fvn_cdevpriv; /* (d) Private data for the cdev. */ struct fadvise_info *fvn_advice; } f_vnun; /* * DFLAG_SEEKABLE specific fields */ off_t f_offset; }; #define f_cdevpriv f_vnun.fvn_cdevpriv #define f_advice f_vnun.fvn_advice #define FOFFSET_LOCKED 0x1 #define FOFFSET_LOCK_WAITING 0x2 #endif /* __BSD_VISIBLE */ #endif /* _KERNEL || _WANT_FILE */ /* * Userland version of struct file, for sysctl */ #if __BSD_VISIBLE struct xfile { ksize_t xf_size; /* size of struct xfile */ pid_t xf_pid; /* owning process */ uid_t xf_uid; /* effective uid of owning process */ int xf_fd; /* descriptor number */ int _xf_int_pad1; kvaddr_t xf_file; /* address of struct file */ short xf_type; /* descriptor type */ short _xf_short_pad1; int xf_count; /* reference count */ int xf_msgcount; /* references from message queue */ int _xf_int_pad2; off_t xf_offset; /* file offset */ kvaddr_t xf_data; /* file descriptor specific data */ kvaddr_t xf_vnode; /* vnode pointer */ u_int xf_flag; /* flags (see fcntl.h) */ int _xf_int_pad3; int64_t _xf_int64_pad[6]; }; #endif /* __BSD_VISIBLE */ #ifdef _KERNEL extern struct fileops vnops; extern struct fileops badfileops; extern struct fileops path_fileops; extern struct fileops socketops; extern int maxfiles; /* kernel limit on number of open files */ extern int maxfilesperproc; /* per process limit on number of open files */ int fget(struct thread *td, int fd, cap_rights_t *rightsp, struct file **fpp); int fget_mmap(struct thread *td, int fd, cap_rights_t *rightsp, vm_prot_t *maxprotp, struct file **fpp); int fget_read(struct thread *td, int fd, cap_rights_t *rightsp, struct file **fpp); int fget_write(struct thread *td, int fd, cap_rights_t *rightsp, struct file **fpp); int fget_fcntl(struct thread *td, int fd, cap_rights_t *rightsp, int needfcntl, struct file **fpp); int _fdrop(struct file *fp, struct thread *td); int fget_remote(struct thread *td, struct proc *p, int fd, struct file **fpp); fo_rdwr_t invfo_rdwr; fo_truncate_t invfo_truncate; fo_ioctl_t invfo_ioctl; fo_poll_t invfo_poll; fo_kqfilter_t invfo_kqfilter; fo_chmod_t invfo_chmod; fo_chown_t invfo_chown; fo_sendfile_t invfo_sendfile; fo_stat_t vn_statfile; fo_sendfile_t vn_sendfile; fo_seek_t vn_seek; fo_fill_kinfo_t vn_fill_kinfo; fo_kqfilter_t vn_kqfilter_opath; int vn_fill_kinfo_vnode(struct vnode *vp, struct kinfo_file *kif); int file_kcmp_generic(struct file *fp1, struct file *fp2, struct thread *td); void finit(struct file *, u_int, short, void *, struct fileops *); void finit_vnode(struct file *, u_int, void *, struct fileops *); int fgetvp(struct thread *td, int fd, cap_rights_t *rightsp, struct vnode **vpp); int fgetvp_exec(struct thread *td, int fd, cap_rights_t *rightsp, struct vnode **vpp); int fgetvp_rights(struct thread *td, int fd, cap_rights_t *needrightsp, struct filecaps *havecaps, struct vnode **vpp); int fgetvp_read(struct thread *td, int fd, cap_rights_t *rightsp, struct vnode **vpp); int fgetvp_write(struct thread *td, int fd, cap_rights_t *rightsp, struct vnode **vpp); int fgetvp_lookup_smr(struct nameidata *ndp, struct vnode **vpp, bool *fsearch); int fgetvp_lookup(struct nameidata *ndp, struct vnode **vpp); static __inline __result_use_check bool fhold(struct file *fp) { return (refcount_acquire_checked(&fp->f_count)); } #define fdrop(fp, td) ({ \ struct file *_fp; \ int _error; \ \ _error = 0; \ _fp = (fp); \ if (__predict_false(refcount_release(&_fp->f_count))) \ _error = _fdrop(_fp, td); \ _error; \ }) #define fdrop_close(fp, td) ({ \ struct file *_fp; \ int _error; \ \ _error = 0; \ _fp = (fp); \ if (__predict_true(refcount_release(&_fp->f_count))) \ _error = _fdrop(_fp, td); \ _error; \ }) static __inline fo_rdwr_t fo_read; static __inline fo_rdwr_t fo_write; static __inline fo_truncate_t fo_truncate; static __inline fo_ioctl_t fo_ioctl; static __inline fo_poll_t fo_poll; static __inline fo_kqfilter_t fo_kqfilter; static __inline fo_stat_t fo_stat; static __inline fo_close_t fo_close; static __inline fo_chmod_t fo_chmod; static __inline fo_chown_t fo_chown; static __inline fo_sendfile_t fo_sendfile; static __inline int fo_read(struct file *fp, struct uio *uio, struct ucred *active_cred, int flags, struct thread *td) { return ((*fp->f_ops->fo_read)(fp, uio, active_cred, flags, td)); } static __inline int fo_write(struct file *fp, struct uio *uio, struct ucred *active_cred, int flags, struct thread *td) { return ((*fp->f_ops->fo_write)(fp, uio, active_cred, flags, td)); } static __inline int fo_truncate(struct file *fp, off_t length, struct ucred *active_cred, struct thread *td) { return ((*fp->f_ops->fo_truncate)(fp, length, active_cred, td)); } static __inline int fo_ioctl(struct file *fp, u_long com, void *data, struct ucred *active_cred, struct thread *td) { return ((*fp->f_ops->fo_ioctl)(fp, com, data, active_cred, td)); } static __inline int fo_poll(struct file *fp, int events, struct ucred *active_cred, struct thread *td) { return ((*fp->f_ops->fo_poll)(fp, events, active_cred, td)); } static __inline int fo_stat(struct file *fp, struct stat *sb, struct ucred *active_cred) { return ((*fp->f_ops->fo_stat)(fp, sb, active_cred)); } static __inline int fo_close(struct file *fp, struct thread *td) { return ((*fp->f_ops->fo_close)(fp, td)); } static __inline int fo_kqfilter(struct file *fp, struct knote *kn) { return ((*fp->f_ops->fo_kqfilter)(fp, kn)); } static __inline int fo_chmod(struct file *fp, mode_t mode, struct ucred *active_cred, struct thread *td) { return ((*fp->f_ops->fo_chmod)(fp, mode, active_cred, td)); } static __inline int fo_chown(struct file *fp, uid_t uid, gid_t gid, struct ucred *active_cred, struct thread *td) { return ((*fp->f_ops->fo_chown)(fp, uid, gid, active_cred, td)); } static __inline int fo_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio, struct uio *trl_uio, off_t offset, size_t nbytes, off_t *sent, int flags, struct thread *td) { return ((*fp->f_ops->fo_sendfile)(fp, sockfd, hdr_uio, trl_uio, offset, nbytes, sent, flags, td)); } static __inline int fo_seek(struct file *fp, off_t offset, int whence, struct thread *td) { return ((*fp->f_ops->fo_seek)(fp, offset, whence, td)); } static __inline int fo_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp) { return ((*fp->f_ops->fo_fill_kinfo)(fp, kif, fdp)); } static __inline int fo_mmap(struct file *fp, vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot, vm_prot_t cap_maxprot, int flags, vm_ooffset_t foff, struct thread *td) { if (fp->f_ops->fo_mmap == NULL) return (ENODEV); return ((*fp->f_ops->fo_mmap)(fp, map, addr, size, prot, cap_maxprot, flags, foff, td)); } static __inline int fo_aio_queue(struct file *fp, struct kaiocb *job) { return ((*fp->f_ops->fo_aio_queue)(fp, job)); } static __inline int fo_add_seals(struct file *fp, int seals) { if (fp->f_ops->fo_add_seals == NULL) return (EINVAL); return ((*fp->f_ops->fo_add_seals)(fp, seals)); } static __inline int fo_get_seals(struct file *fp, int *seals) { if (fp->f_ops->fo_get_seals == NULL) return (EINVAL); return ((*fp->f_ops->fo_get_seals)(fp, seals)); } static __inline int fo_fallocate(struct file *fp, off_t offset, off_t len, struct thread *td) { if (fp->f_ops->fo_fallocate == NULL) return (ENODEV); return ((*fp->f_ops->fo_fallocate)(fp, offset, len, td)); } static __inline int fo_fspacectl(struct file *fp, int cmd, off_t *offset, off_t *length, int flags, struct ucred *active_cred, struct thread *td) { if (fp->f_ops->fo_fspacectl == NULL) return (ENODEV); return ((*fp->f_ops->fo_fspacectl)(fp, cmd, offset, length, flags, active_cred, td)); } static __inline int fo_cmp(struct file *fp1, struct file *fp2, struct thread *td) { if (fp1->f_ops->fo_cmp == NULL) return (ENODEV); return ((*fp1->f_ops->fo_cmp)(fp1, fp2, td)); } #endif /* _KERNEL */ #endif /* !SYS_FILE_H */ diff --git a/sys/sys/ktls.h b/sys/sys/ktls.h index 9b3433f4b1fd..0e52d643fe3b 100644 --- a/sys/sys/ktls.h +++ b/sys/sys/ktls.h @@ -1,259 +1,262 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014-2019 Netflix Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _SYS_KTLS_H_ #define _SYS_KTLS_H_ #ifdef _KERNEL +#include #include #include + +#include #endif struct tls_record_layer { uint8_t tls_type; uint8_t tls_vmajor; uint8_t tls_vminor; uint16_t tls_length; uint8_t tls_data[0]; } __attribute__ ((packed)); #define TLS_MAX_MSG_SIZE_V10_2 16384 #define TLS_MAX_PARAM_SIZE 1024 /* Max key/mac/iv in sockopt */ #define TLS_AEAD_GCM_LEN 4 #define TLS_1_3_GCM_IV_LEN 12 #define TLS_CHACHA20_IV_LEN 12 #define TLS_CBC_IMPLICIT_IV_LEN 16 /* Type values for the record layer */ #define TLS_RLTYPE_ALERT 21 #define TLS_RLTYPE_HANDSHAKE 22 #define TLS_RLTYPE_APP 23 /* * Nonce for GCM for TLS 1.2 per RFC 5288. */ struct tls_nonce_data { uint8_t fixed[TLS_AEAD_GCM_LEN]; uint64_t seq; } __packed; /* * AEAD additional data format for TLS 1.2 per RFC 5246. */ struct tls_aead_data { uint64_t seq; /* In network order */ uint8_t type; uint8_t tls_vmajor; uint8_t tls_vminor; uint16_t tls_length; } __packed; /* * AEAD additional data format for TLS 1.3 per RFC 8446. */ struct tls_aead_data_13 { uint8_t type; uint8_t tls_vmajor; uint8_t tls_vminor; uint16_t tls_length; } __packed; /* * Stream Cipher MAC additional data input. This does not match the * exact data on the wire (the sequence number is not placed on the * wire, and any explicit IV after the record header is not covered by * the MAC). */ struct tls_mac_data { uint64_t seq; uint8_t type; uint8_t tls_vmajor; uint8_t tls_vminor; uint16_t tls_length; } __packed; #define TLS_MAJOR_VER_ONE 3 #define TLS_MINOR_VER_ZERO 1 /* 3, 1 */ #define TLS_MINOR_VER_ONE 2 /* 3, 2 */ #define TLS_MINOR_VER_TWO 3 /* 3, 3 */ #define TLS_MINOR_VER_THREE 4 /* 3, 4 */ /* For TCP_TXTLS_ENABLE and TCP_RXTLS_ENABLE. */ #ifdef _KERNEL struct tls_enable_v0 { const uint8_t *cipher_key; const uint8_t *iv; /* Implicit IV. */ const uint8_t *auth_key; int cipher_algorithm; /* e.g. CRYPTO_AES_CBC */ int cipher_key_len; int iv_len; int auth_algorithm; /* e.g. CRYPTO_SHA2_256_HMAC */ int auth_key_len; int flags; uint8_t tls_vmajor; uint8_t tls_vminor; }; #endif struct tls_enable { const uint8_t *cipher_key; const uint8_t *iv; /* Implicit IV. */ const uint8_t *auth_key; int cipher_algorithm; /* e.g. CRYPTO_AES_CBC */ int cipher_key_len; int iv_len; int auth_algorithm; /* e.g. CRYPTO_SHA2_256_HMAC */ int auth_key_len; int flags; uint8_t tls_vmajor; uint8_t tls_vminor; uint8_t rec_seq[8]; }; /* Structure for TLS_GET_RECORD. */ struct tls_get_record { /* TLS record header. */ uint8_t tls_type; uint8_t tls_vmajor; uint8_t tls_vminor; uint16_t tls_length; }; #ifdef _KERNEL struct tls_session_params { uint8_t *cipher_key; uint8_t *auth_key; uint8_t iv[TLS_CBC_IMPLICIT_IV_LEN]; int cipher_algorithm; int auth_algorithm; uint16_t cipher_key_len; uint16_t iv_len; uint16_t auth_key_len; uint16_t max_frame_len; uint8_t tls_vmajor; uint8_t tls_vminor; uint8_t tls_hlen; uint8_t tls_tlen; uint8_t tls_bs; uint8_t flags; }; /* Used in APIs to request RX vs TX sessions. */ #define KTLS_TX 1 #define KTLS_RX 2 struct iovec; struct ktls_ocf_encrypt_state; struct ktls_ocf_session; struct ktls_session; struct m_snd_tag; struct mbuf; struct sockbuf; struct socket; struct sockopt; struct ktls_session { struct ktls_ocf_session *ocf_session; struct m_snd_tag *snd_tag; struct tls_session_params params; u_int wq_index; volatile u_int refcount; int mode; struct task reset_tag_task; struct task disable_ifnet_task; union { struct inpcb *inp; /* Used by transmit tasks. */ struct socket *so; /* Used by receive task. */ }; struct ifnet *rx_ifp; u_short rx_vlan_id; bool reset_pending; bool tx; bool sync_dispatch; bool sequential_records; /* Only used for TLS 1.0. */ uint64_t next_seqno; STAILQ_HEAD(, mbuf) pending_records; /* Used to destroy any kTLS session */ struct task destroy_task; } __aligned(CACHE_LINE_SIZE); extern unsigned int ktls_ifnet_max_rexmit_pct; typedef enum { KTLS_MBUF_CRYPTO_ST_MIXED = 0, KTLS_MBUF_CRYPTO_ST_ENCRYPTED = 1, KTLS_MBUF_CRYPTO_ST_DECRYPTED = -1, } ktls_mbuf_crypto_st_t; void ktls_check_rx(struct sockbuf *sb); void ktls_cleanup_tls_enable(struct tls_enable *tls); int ktls_copyin_tls_enable(struct sockopt *sopt, struct tls_enable *tls); void ktls_disable_ifnet(void *arg); int ktls_enable_rx(struct socket *so, struct tls_enable *en); int ktls_enable_tx(struct socket *so, struct tls_enable *en); void ktls_enqueue(struct mbuf *m, struct socket *so, int page_count); void ktls_enqueue_to_free(struct mbuf *m); void ktls_destroy(struct ktls_session *tls); void ktls_frame(struct mbuf *m, struct ktls_session *tls, int *enqueue_cnt, uint8_t record_type); int ktls_get_rx_mode(struct socket *so, int *modep); int ktls_get_tx_mode(struct socket *so, int *modep); int ktls_get_rx_sequence(struct inpcb *inp, uint32_t *tcpseq, uint64_t *tlsseq); void ktls_input_ifp_mismatch(struct sockbuf *sb, struct ifnet *ifp); ktls_mbuf_crypto_st_t ktls_mbuf_crypto_state(struct mbuf *mb, int offset, int len); #ifdef RATELIMIT int ktls_modify_txrtlmt(struct ktls_session *tls, uint64_t max_pacing_rate); #endif int ktls_output_eagain(struct inpcb *inp, struct ktls_session *tls); bool ktls_pending_rx_info(struct sockbuf *sb, uint64_t *seqnop, size_t *residp); bool ktls_permit_empty_frames(struct ktls_session *tls); void ktls_seq(struct sockbuf *sb, struct mbuf *m); int ktls_set_tx_mode(struct socket *so, int mode); static inline struct ktls_session * ktls_hold(struct ktls_session *tls) { if (tls != NULL) refcount_acquire(&tls->refcount); return (tls); } static inline void ktls_free(struct ktls_session *tls) { if (refcount_release(&tls->refcount)) ktls_destroy(tls); } #endif /* !_KERNEL */ #endif /* !_SYS_KTLS_H_ */ diff --git a/sys/sys/refcount.h b/sys/sys/refcount.h index 44588fe3c812..ae0ec44fd7a6 100644 --- a/sys/sys/refcount.h +++ b/sys/sys/refcount.h @@ -1,219 +1,220 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2005 John Baldwin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef __SYS_REFCOUNT_H__ #define __SYS_REFCOUNT_H__ -#include - -#include +#include +#include #if !defined(_KERNEL) && !defined(_STANDALONE) #include #endif +#include + #define REFCOUNT_SATURATED(val) (((val) & (1U << 31)) != 0) #define REFCOUNT_SATURATION_VALUE (3U << 30) /* * Attempt to handle reference count overflow and underflow. Force the counter * to stay at the saturation value so that a counter overflow cannot trigger * destruction of the containing object and instead leads to a less harmful * memory leak. */ static __inline void _refcount_update_saturated(volatile u_int *count) { #ifdef INVARIANTS panic("refcount %p wraparound", count); #else atomic_store_int(count, REFCOUNT_SATURATION_VALUE); #endif } static __inline void refcount_init(volatile u_int *count, u_int value) { KASSERT(!REFCOUNT_SATURATED(value), ("invalid initial refcount value %u", value)); atomic_store_int(count, value); } static __inline u_int refcount_load(volatile u_int *count) { return (atomic_load_int(count)); } static __inline u_int refcount_acquire(volatile u_int *count) { u_int old; old = atomic_fetchadd_int(count, 1); if (__predict_false(REFCOUNT_SATURATED(old))) _refcount_update_saturated(count); return (old); } static __inline u_int refcount_acquiren(volatile u_int *count, u_int n) { u_int old; KASSERT(n < REFCOUNT_SATURATION_VALUE / 2, ("refcount_acquiren: n=%u too large", n)); old = atomic_fetchadd_int(count, n); if (__predict_false(REFCOUNT_SATURATED(old))) _refcount_update_saturated(count); return (old); } static __inline __result_use_check bool refcount_acquire_checked(volatile u_int *count) { u_int old; old = atomic_load_int(count); for (;;) { if (__predict_false(REFCOUNT_SATURATED(old + 1))) return (false); if (__predict_true(atomic_fcmpset_int(count, &old, old + 1) == 1)) return (true); } } /* * This functions returns non-zero if the refcount was * incremented. Else zero is returned. */ static __inline __result_use_check bool refcount_acquire_if_gt(volatile u_int *count, u_int n) { u_int old; old = atomic_load_int(count); for (;;) { if (old <= n) return (false); if (__predict_false(REFCOUNT_SATURATED(old))) return (true); if (atomic_fcmpset_int(count, &old, old + 1)) return (true); } } static __inline __result_use_check bool refcount_acquire_if_not_zero(volatile u_int *count) { return (refcount_acquire_if_gt(count, 0)); } static __inline bool refcount_releasen(volatile u_int *count, u_int n) { u_int old; KASSERT(n < REFCOUNT_SATURATION_VALUE / 2, ("refcount_releasen: n=%u too large", n)); atomic_thread_fence_rel(); old = atomic_fetchadd_int(count, -n); if (__predict_false(old < n || REFCOUNT_SATURATED(old))) { _refcount_update_saturated(count); return (false); } if (old > n) return (false); /* * Last reference. Signal the user to call the destructor. * * Ensure that the destructor sees all updates. This synchronizes with * release fences from all routines which drop the count. */ atomic_thread_fence_acq(); return (true); } static __inline bool refcount_release(volatile u_int *count) { return (refcount_releasen(count, 1)); } #define _refcount_release_if_cond(cond, name) \ static __inline __result_use_check bool \ _refcount_release_if_##name(volatile u_int *count, u_int n) \ { \ u_int old; \ \ KASSERT(n > 0, ("%s: zero increment", __func__)); \ old = atomic_load_int(count); \ for (;;) { \ if (!(cond)) \ return (false); \ if (__predict_false(REFCOUNT_SATURATED(old))) \ return (false); \ if (atomic_fcmpset_rel_int(count, &old, old - 1)) \ return (true); \ } \ } _refcount_release_if_cond(old > n, gt) _refcount_release_if_cond(old == n, eq) static __inline __result_use_check bool refcount_release_if_gt(volatile u_int *count, u_int n) { return (_refcount_release_if_gt(count, n)); } static __inline __result_use_check bool refcount_release_if_last(volatile u_int *count) { if (_refcount_release_if_eq(count, 1)) { /* See the comment in refcount_releasen(). */ atomic_thread_fence_acq(); return (true); } return (false); } static __inline __result_use_check bool refcount_release_if_not_last(volatile u_int *count) { return (_refcount_release_if_gt(count, 1)); } #endif /* !__SYS_REFCOUNT_H__ */ diff --git a/sys/sys/sglist.h b/sys/sys/sglist.h index 96f3c1cd5ffb..3bbada573772 100644 --- a/sys/sys/sglist.h +++ b/sys/sys/sglist.h @@ -1,115 +1,116 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 2008 Yahoo!, Inc. * All rights reserved. * Written by: John Baldwin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the author nor the names of any co-contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * A scatter/gather list describes a group of physical address ranges. * Each physical address range consists of a starting address and a * length. */ #ifndef __SGLIST_H__ #define __SGLIST_H__ #include struct sglist_seg { vm_paddr_t ss_paddr; size_t ss_len; }; struct sglist { struct sglist_seg *sg_segs; u_int sg_refs; u_short sg_nseg; u_short sg_maxseg; }; struct bio; struct mbuf; +struct thread; struct uio; static __inline void sglist_init(struct sglist *sg, u_short maxsegs, struct sglist_seg *segs) { sg->sg_segs = segs; sg->sg_nseg = 0; sg->sg_maxseg = maxsegs; refcount_init(&sg->sg_refs, 1); } static __inline void sglist_reset(struct sglist *sg) { sg->sg_nseg = 0; } static __inline struct sglist * sglist_hold(struct sglist *sg) { refcount_acquire(&sg->sg_refs); return (sg); } struct sglist *sglist_alloc(int nsegs, int mflags); int sglist_append(struct sglist *sg, void *buf, size_t len); int sglist_append_bio(struct sglist *sg, struct bio *bp); int sglist_append_mbuf(struct sglist *sg, struct mbuf *m0); int sglist_append_mbuf_epg(struct sglist *sg, struct mbuf *m0, size_t off, size_t len); int sglist_append_phys(struct sglist *sg, vm_paddr_t paddr, size_t len); int sglist_append_sglist(struct sglist *sg, struct sglist *source, size_t offset, size_t length); int sglist_append_single_mbuf(struct sglist *sg, struct mbuf *m); int sglist_append_uio(struct sglist *sg, struct uio *uio); int sglist_append_user(struct sglist *sg, void *buf, size_t len, struct thread *td); int sglist_append_vmpages(struct sglist *sg, vm_page_t *m, size_t pgoff, size_t len); struct sglist *sglist_build(void *buf, size_t len, int mflags); struct sglist *sglist_clone(struct sglist *sg, int mflags); int sglist_consume_uio(struct sglist *sg, struct uio *uio, size_t resid); int sglist_count(void *buf, size_t len); int sglist_count_mbuf_epg(struct mbuf *m, size_t off, size_t len); int sglist_count_vmpages(vm_page_t *m, size_t pgoff, size_t len); void sglist_free(struct sglist *sg); int sglist_join(struct sglist *first, struct sglist *second); size_t sglist_length(struct sglist *sg); int sglist_slice(struct sglist *original, struct sglist **slice, size_t offset, size_t length, int mflags); int sglist_split(struct sglist *original, struct sglist **head, size_t length, int mflags); #endif /* !__SGLIST_H__ */ diff --git a/tools/build/test-includes/badfiles.inc b/tools/build/test-includes/badfiles.inc index eb0cd05cb39f..1d648819cd8c 100644 --- a/tools/build/test-includes/badfiles.inc +++ b/tools/build/test-includes/badfiles.inc @@ -1,326 +1,325 @@ # DO NOT EDIT-- this file is automatically @generated. BADHDRS= \ sys/aac_ioctl.h \ sys/agpio.h \ sys/atomic_common.h \ sys/atomic_san.h \ sys/bio.h \ sys/bitstring.h \ sys/blist.h \ sys/buf.h \ sys/buf_ring.h \ sys/bus.h \ sys/bus_dma.h \ sys/bus_dma_internal.h \ sys/bus_san.h \ sys/busdma_bufalloc.h \ sys/caprights.h \ sys/cdrio.h \ sys/cfictl.h \ sys/conf.h \ sys/cons.h \ sys/copyright.h \ sys/counter.h \ sys/coverage.h \ sys/cpu.h \ sys/cpuctl.h \ sys/cpuset.h \ sys/devmap.h \ sys/disk_zone.h \ sys/domain.h \ sys/domainset.h \ sys/dtrace_bsd.h \ sys/dvdio.h \ sys/elf32.h \ sys/elf64.h \ sys/elf_common.h \ sys/elf_generic.h \ sys/eui64.h \ sys/eventhandler.h \ sys/eventvar.h \ sys/exec.h \ sys/extattr.h \ sys/fail.h \ sys/firmware.h \ sys/fnv_hash.h \ sys/font.h \ sys/gmon.h \ sys/gpio.h \ sys/gtaskqueue.h \ sys/hhook.h \ sys/iconv.h \ sys/imgact.h \ sys/imgact_aout.h \ sys/imgact_elf.h \ sys/interrupt.h \ sys/intr.h \ sys/ioctl_compat.h \ sys/iov.h \ sys/iov_schema.h \ sys/ipmi.h \ sys/jail.h \ sys/kdb.h \ sys/kernel.h \ sys/khelp.h \ sys/kobj.h \ sys/ksem.h \ sys/ktls.h \ sys/ktrace.h \ sys/libkern.h \ sys/link_aout.h \ sys/linker.h \ sys/linker_set.h \ sys/lock.h \ sys/lock_profile.h \ sys/lockf.h \ sys/lockmgr.h \ sys/loginclass.h \ sys/mac.h \ sys/mbuf.h \ sys/md4.h \ sys/md5.h \ sys/mdioctl.h \ sys/memdesc.h \ sys/memrange.h \ sys/module.h \ sys/module_khelp.h \ sys/mpt_ioctl.h \ sys/msgbuf.h \ sys/mutex.h \ sys/namei.h \ sys/osd.h \ sys/pciio.h \ sys/pctrie.h \ sys/physmem.h \ sys/pipe.h \ sys/pmckern.h \ sys/posix4.h \ sys/power.h \ sys/priority.h \ sys/prng.h \ sys/qmath.h \ sys/racct.h \ - sys/refcount.h \ sys/resourcevar.h \ sys/rman.h \ sys/rmlock.h \ sys/rtprio.h \ sys/runq.h \ sys/rwlock.h \ sys/sbuf.h \ sys/sema.h \ sys/sf_buf.h \ sys/sglist.h \ sys/sigio.h \ sys/signalvar.h \ sys/smr.h \ sys/smr_types.h \ sys/socketvar.h \ sys/sockopt.h \ sys/stack.h \ sys/stats.h \ sys/sx.h \ sys/syscallsubr.h \ sys/syslimits.h \ sys/systm.h \ sys/taskqueue.h \ sys/terminal.h \ sys/termios.h \ sys/tiio.h \ sys/timeb.h \ sys/timeet.h \ sys/timeffc.h \ sys/timepps.h \ sys/timetc.h \ sys/ttydevsw.h \ sys/ttydisc.h \ sys/ttyhook.h \ sys/ttyqueue.h \ sys/umtx.h \ sys/unpcb.h \ sys/user.h \ sys/vdso.h \ sys/vmmeter.h \ sys/vnode.h \ net/bpf.h \ net/bpf_buffer.h \ net/bpf_jitter.h \ net/bpf_zerocopy.h \ net/bpfdesc.h \ net/debugnet_int.h \ net/firewire.h \ net/if_arp.h \ net/if_dl.h \ net/if_enc.h \ net/if_lagg.h \ net/if_llatbl.h \ net/if_llc.h \ net/if_media.h \ net/if_mib.h \ net/if_pfsync.h \ net/if_tap.h \ net/if_tun.h \ net/if_var.h \ net/if_vlan_var.h \ net/iflib.h \ net/iflib_private.h \ net/ifq.h \ net/mp_ring.h \ net/mppc.h \ net/netisr.h \ net/netisr_internal.h \ net/netmap.h \ net/netmap_legacy.h \ net/paravirt.h \ net/pfil.h \ net/pfkeyv2.h \ net/pfvar.h \ net/radix.h \ net/raw_cb.h \ net/rndis.h \ net/route.h \ net/rss_config.h \ net/slcompress.h \ net/toeplitz.h \ net80211/ieee80211.h \ net80211/ieee80211_action.h \ net80211/ieee80211_adhoc.h \ net80211/ieee80211_ageq.h \ net80211/ieee80211_alq.h \ net80211/ieee80211_amrr.h \ net80211/ieee80211_crypto.h \ net80211/ieee80211_dfs.h \ net80211/ieee80211_freebsd.h \ net80211/ieee80211_hostap.h \ net80211/ieee80211_ht.h \ net80211/ieee80211_input.h \ net80211/ieee80211_ioctl.h \ net80211/ieee80211_mesh.h \ net80211/ieee80211_monitor.h \ net80211/ieee80211_node.h \ net80211/ieee80211_power.h \ net80211/ieee80211_proto.h \ net80211/ieee80211_radiotap.h \ net80211/ieee80211_ratectl.h \ net80211/ieee80211_rssadapt.h \ net80211/ieee80211_scan.h \ net80211/ieee80211_scan_sw.h \ net80211/ieee80211_sta.h \ net80211/ieee80211_superg.h \ net80211/ieee80211_tdma.h \ net80211/ieee80211_var.h \ net80211/ieee80211_vht.h \ net80211/ieee80211_wds.h \ netgraph/netgraph.h \ netgraph/ng_async.h \ netgraph/ng_bpf.h \ netgraph/ng_bridge.h \ netgraph/ng_car.h \ netgraph/ng_checksum.h \ netgraph/ng_cisco.h \ netgraph/ng_deflate.h \ netgraph/ng_etf.h \ netgraph/ng_hole.h \ netgraph/ng_ksocket.h \ netgraph/ng_l2tp.h \ netgraph/ng_lmi.h \ netgraph/ng_macfilter.h \ netgraph/ng_message.h \ netgraph/ng_mppc.h \ netgraph/ng_nat.h \ netgraph/ng_one2many.h \ netgraph/ng_parse.h \ netgraph/ng_patch.h \ netgraph/ng_pipe.h \ netgraph/ng_ppp.h \ netgraph/ng_pppoe.h \ netgraph/ng_pptpgre.h \ netgraph/ng_pred1.h \ netgraph/ng_sample.h \ netgraph/ng_socket.h \ netgraph/ng_socketvar.h \ netgraph/ng_source.h \ netgraph/ng_tag.h \ netgraph/ng_tcpmss.h \ netgraph/ng_tee.h \ netgraph/ng_vjc.h \ netgraph/ng_vlan.h \ netgraph/ng_vlan_rotate.h \ netgraph/qos.h \ netinet/dccp.h \ netinet/icmp6.h \ netinet/icmp_var.h \ netinet/if_ether.h \ netinet/igmp.h \ netinet/igmp_var.h \ netinet/in_fib.h \ netinet/in_kdtrace.h \ netinet/in_pcb.h \ netinet/in_pcb_var.h \ netinet/in_rss.h \ netinet/in_systm.h \ netinet/in_var.h \ netinet/ip.h \ netinet/ip6.h \ netinet/ip_carp.h \ netinet/ip_dummynet.h \ netinet/ip_fw.h \ netinet/ip_icmp.h \ netinet/ip_mroute.h \ netinet/ip_options.h \ netinet/pim_var.h \ netinet/sctp_auth.h \ netinet/sctp_bsd_addr.h \ netinet/sctp_kdtrace.h \ netinet/sctp_os.h \ netinet/sctp_os_bsd.h \ netinet/sctp_pcb.h \ netinet/sctp_structs.h \ netinet/sctp_sysctl.h \ netinet/tcp_debug.h \ netinet/tcp_hpts.h \ netinet/tcp_log_buf.h \ netinet/tcp_lro.h \ netinet/tcp_offload.h \ netinet/tcp_pcap.h \ netinet/tcp_ratelimit.h \ netinet/tcp_var.h \ netinet/tcpip.h \ netinet/toecore.h \ netinet/udplite.h \ netinet6/icmp6.h \ netinet6/in6.h \ netinet6/in6_fib.h \ netinet6/in6_rss.h \ netinet6/in6_var.h \ netinet6/ip6.h \ netinet6/ip6_mroute.h \ netinet6/ip6_var.h \ netinet6/ip6protosw.h \ netinet6/ip_fw_nat64.h \ netinet6/ip_fw_nptv6.h \ netinet6/mld6.h \ netinet6/mld6_var.h \ netinet6/nd6.h \ netinet6/pim6.h \ netinet6/pim6_var.h \ netinet6/raw_ip6.h \ netinet6/send.h \ netipsec/ah.h \ netipsec/ah_var.h \ netipsec/esp.h \ netipsec/esp_var.h \ netipsec/ipcomp.h \ netipsec/ipcomp_var.h \ netipsec/ipsec.h \ netipsec/ipsec6.h \ netipsec/key_debug.h \ netipsec/keysock.h \ netipsec/xform.h \ netsmb/netbios.h \ netsmb/smb.h \ netsmb/smb_conn.h \ netsmb/smb_rq.h \ netsmb/smb_subr.h \ netsmb/smb_tran.h \