Index: head/share/man/man9/sleep.9
===================================================================
--- head/share/man/man9/sleep.9 (revision 356056)
+++ head/share/man/man9/sleep.9 (revision 356057)
@@ -1,423 +1,423 @@
.\"
.\" Copyright (c) 1996 Joerg Wunsch
.\"
.\" All rights reserved.
.\"
.\" Redistribution and use in source and binary forms, with or without
.\" modification, are permitted provided that the following conditions
.\" are met:
.\" 1. Redistributions of source code must retain the above copyright
.\" notice, this list of conditions and the following disclaimer.
.\" 2. Redistributions in binary form must reproduce the above copyright
.\" notice, this list of conditions and the following disclaimer in the
.\" documentation and/or other materials provided with the distribution.
.\"
.\" THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY EXPRESS OR
.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
.\" OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
.\" IN NO EVENT SHALL THE DEVELOPERS BE LIABLE FOR ANY DIRECT, INDIRECT,
.\" INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
.\" NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
.\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
.\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
.\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
.\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.\"
.\" $FreeBSD$
.\"
.Dd June 19, 2019
.Dt SLEEP 9
.Os
.Sh NAME
.Nm msleep ,
.Nm msleep_sbt ,
.Nm msleep_spin ,
.Nm msleep_spin_sbt ,
.Nm pause ,
.Nm pause_sig ,
.Nm pause_sbt ,
.Nm tsleep ,
.Nm tsleep_sbt ,
.Nm wakeup ,
.Nm wakeup_one ,
.Nm wakeup_any
.Nd wait for events
.Sh SYNOPSIS
.In sys/param.h
.In sys/systm.h
.In sys/proc.h
.Ft int
-.Fn msleep "void *chan" "struct mtx *mtx" "int priority" "const char *wmesg" "int timo"
+.Fn msleep "const void *chan" "struct mtx *mtx" "int priority" "const char *wmesg" "int timo"
.Ft int
-.Fn msleep_sbt "void *chan" "struct mtx *mtx" "int priority" \
+.Fn msleep_sbt "const void *chan" "struct mtx *mtx" "int priority" \
"const char *wmesg" "sbintime_t sbt" "sbintime_t pr" "int flags"
.Ft int
-.Fn msleep_spin "void *chan" "struct mtx *mtx" "const char *wmesg" "int timo"
+.Fn msleep_spin "const void *chan" "struct mtx *mtx" "const char *wmesg" "int timo"
.Ft int
-.Fn msleep_spin_sbt "void *chan" "struct mtx *mtx" "const char *wmesg" \
+.Fn msleep_spin_sbt "const void *chan" "struct mtx *mtx" "const char *wmesg" \
"sbintime_t sbt" "sbintime_t pr" "int flags"
.Ft int
.Fn pause "const char *wmesg" "int timo"
.Ft int
.Fn pause_sig "const char *wmesg" "int timo"
.Ft int
.Fn pause_sbt "const char *wmesg" "sbintime_t sbt" "sbintime_t pr" \
"int flags"
.Ft int
-.Fn tsleep "void *chan" "int priority" "const char *wmesg" "int timo"
+.Fn tsleep "const void *chan" "int priority" "const char *wmesg" "int timo"
.Ft int
-.Fn tsleep_sbt "void *chan" "int priority" "const char *wmesg" \
+.Fn tsleep_sbt "const void *chan" "int priority" "const char *wmesg" \
"sbintime_t sbt" "sbintime_t pr" "int flags"
.Ft void
-.Fn wakeup "void *chan"
+.Fn wakeup "const void *chan"
.Ft void
-.Fn wakeup_one "void *chan"
+.Fn wakeup_one "const void *chan"
.Ft void
-.Fn wakeup_any "void *chan"
+.Fn wakeup_any "const void *chan"
.Sh DESCRIPTION
The functions
.Fn tsleep ,
.Fn msleep ,
.Fn msleep_spin ,
.Fn pause ,
.Fn pause_sig ,
.Fn pause_sbt ,
.Fn wakeup ,
.Fn wakeup_one ,
and
.Fn wakeup_any
handle event-based thread blocking.
If a thread must wait for an
external event, it is put to sleep by
.Fn tsleep ,
.Fn msleep ,
.Fn msleep_spin ,
.Fn pause ,
.Fn pause_sig ,
or
.Fn pause_sbt .
Threads may also wait using one of the locking primitive sleep routines
.Xr mtx_sleep 9 ,
.Xr rw_sleep 9 ,
or
.Xr sx_sleep 9 .
.Pp
The parameter
.Fa chan
is an arbitrary address that uniquely identifies the event on which
the thread is being put to sleep.
All threads sleeping on a single
.Fa chan
are woken up later by
.Fn wakeup ,
often called from inside an interrupt routine, to indicate that the
resource the thread was blocking on is available now.
.Pp
The parameter
.Fa priority
specifies a new priority for the thread as well as some optional flags.
If the new priority is not 0,
then the thread will be made
runnable with the specified
.Fa priority
when it resumes.
.Dv PZERO
should never be used, as it is for compatibility only.
A new priority of 0 means to use the thread's current priority when
it is made runnable again.
.Pp
If
.Fa priority
includes the
.Dv PCATCH
flag, pending signals are allowed to interrupt the sleep, otherwise
pending signals are ignored during the sleep.
If
.Dv PCATCH
is set and a signal becomes pending,
.Er ERESTART
is returned if the current system call should be restarted if
possible, and
.Er EINTR
is returned if the system call should be interrupted by the signal
(return
.Er EINTR ) .
.Pp
The parameter
.Fa wmesg
is a string describing the sleep condition for tools like
.Xr ps 1 .
Due to the limited space of those programs to display arbitrary strings,
this message should not be longer than 6 characters.
.Pp
The parameter
.Fa timo
specifies a timeout for the sleep.
If
.Fa timo
is not 0,
then the thread will sleep for at most
.Fa timo No / Va hz
seconds.
If the timeout expires,
then the sleep function will return
.Er EWOULDBLOCK .
.Pp
.Fn msleep_sbt ,
.Fn msleep_spin_sbt ,
.Fn pause_sbt
and
.Fn tsleep_sbt
functions take
.Fa sbt
parameter instead of
.Fa timo .
It allows the caller to specify relative or absolute wakeup time with higher resolution
in form of
.Vt sbintime_t .
The parameter
.Fa pr
allows the caller to specify wanted absolute event precision.
The parameter
.Fa flags
allows the caller to pass additional
.Fn callout_reset_sbt
flags.
.Pp
Several of the sleep functions including
.Fn msleep ,
.Fn msleep_spin ,
and the locking primitive sleep routines specify an additional lock
parameter.
The lock will be released before sleeping and reacquired
before the sleep routine returns.
If
.Fa priority
includes the
.Dv PDROP
flag, then
the lock will not be reacquired before returning.
The lock is used to ensure that a condition can be checked atomically,
and that the current thread can be suspended without missing a
change to the condition, or an associated wakeup.
In addition, all of the sleep routines will fully drop the
.Va Giant
mutex
(even if recursed)
while the thread is suspended and will reacquire the
.Va Giant
mutex before the function returns.
Note that the
.Va Giant
mutex may be specified as the lock to drop.
In that case, however, the
.Dv PDROP
flag is not allowed.
.Pp
To avoid lost wakeups,
either a lock should be used to protect against races,
or a timeout should be specified to place an upper bound on the delay due
to a lost wakeup.
As a result,
the
.Fn tsleep
function should only be invoked with a timeout of 0 when the
.Va Giant
mutex is held.
.Pp
The
.Fn msleep
function requires that
.Fa mtx
reference a default, i.e. non-spin, mutex.
Its use is deprecated in favor of
.Xr mtx_sleep 9
which provides identical behavior.
.Pp
The
.Fn msleep_spin
function requires that
.Fa mtx
reference a spin mutex.
The
.Fn msleep_spin
function does not accept a
.Fa priority
parameter and thus does not support changing the current thread's priority,
the
.Dv PDROP
flag,
or catching signals via the
.Dv PCATCH
flag.
.Pp
The
.Fn pause
function is a wrapper around
.Fn tsleep
that suspends execution of the current thread for the indicated timeout.
The thread can not be awakened early by signals or calls to
.Fn wakeup ,
.Fn wakeup_one
or
.Fn wakeup_any .
The
.Fn pause_sig
function is a variant of
.Fn pause
which can be awakened early by signals.
.Pp
The
.Fn wakeup_one
function makes the first highest priority thread in the queue that is
sleeping on the parameter
.Fa chan
runnable.
This reduces the load when a large number of threads are sleeping on
the same address, but only one of them can actually do any useful work
when made runnable.
.Pp
Due to the way it works, the
.Fn wakeup_one
function requires that only related threads sleep on a specific
.Fa chan
address.
It is the programmer's responsibility to choose a unique
.Fa chan
value.
The older
.Fn wakeup
function did not require this, though it was never good practice
for threads to share a
.Fa chan
value.
When converting from
.Fn wakeup
to
.Fn wakeup_one ,
pay particular attention to ensure that no other threads wait on the
same
.Fa chan .
.Pp
The
.Fn wakeup_any
function is similar to
.Fn wakeup_one ,
except that it makes runnable last thread on the queue (sleeping less),
ignoring fairness.
It can be used when threads sleeping on the
.Fa chan
are known to be identical and there is no reason to be fair.
.Pp
If the timeout given by
.Fa timo
or
.Fa sbt
is based on an absolute real-time clock value,
then the thread should copy the global
.Va rtc_generation
into its
.Va td_rtcgen
member before reading the RTC.
If the real-time clock is adjusted, these functions will set
.Va td_rtcgen
to zero and return zero.
The caller should reconsider its orientation with the new RTC value.
.Sh RETURN VALUES
When awakened by a call to
.Fn wakeup
or
.Fn wakeup_one ,
if a signal is pending and
.Dv PCATCH
is specified,
a non-zero error code is returned.
If the thread is awakened by a call to
.Fn wakeup
or
.Fn wakeup_one ,
the
.Fn msleep ,
.Fn msleep_spin ,
.Fn tsleep ,
and locking primitive sleep functions return 0.
Zero can also be returned when the real-time clock is adjusted;
see above regarding
.Va td_rtcgen .
Otherwise, a non-zero error code is returned.
.Sh ERRORS
.Fn msleep ,
.Fn msleep_spin ,
.Fn tsleep ,
and the locking primitive sleep functions will fail if:
.Bl -tag -width Er
.It Bq Er EINTR
The
.Dv PCATCH
flag was specified, a signal was caught, and the system call should be
interrupted.
.It Bq Er ERESTART
The
.Dv PCATCH
flag was specified, a signal was caught, and the system call should be
restarted.
.It Bq Er EWOULDBLOCK
A non-zero timeout was specified and the timeout expired.
.El
.Sh SEE ALSO
.Xr ps 1 ,
.Xr locking 9 ,
.Xr malloc 9 ,
.Xr mi_switch 9 ,
.Xr mtx_sleep 9 ,
.Xr rw_sleep 9 ,
.Xr sx_sleep 9 ,
.Xr timeout 9
.Sh HISTORY
The functions
.Fn sleep
and
.Fn wakeup
were present in
.At v1 .
They were probably also present in the preceding
PDP-7 version of
.Ux .
They were the basic process synchronization model.
.Pp
The
.Fn tsleep
function appeared in
.Bx 4.4
and added the parameters
.Fa wmesg
and
.Fa timo .
The
.Fn sleep
function was removed in
.Fx 2.2 .
The
.Fn wakeup_one
function appeared in
.Fx 2.2 .
The
.Fn msleep
function appeared in
.Fx 5.0 ,
and the
.Fn msleep_spin
function appeared in
.Fx 6.2 .
The
.Fn pause
function appeared in
.Fx 7.0 .
The
.Fn pause_sig
function appeared in
.Fx 12.0 .
.Sh AUTHORS
.An -nosplit
This manual page was written by
.An J\(:org Wunsch Aq Mt joerg@FreeBSD.org .
Index: head/share/man/man9/sleepqueue.9
===================================================================
--- head/share/man/man9/sleepqueue.9 (revision 356056)
+++ head/share/man/man9/sleepqueue.9 (revision 356057)
@@ -1,390 +1,390 @@
.\" Copyright (c) 2000-2004 John H. Baldwin <jhb@FreeBSD.org>
.\"
.\" Redistribution and use in source and binary forms, with or without
.\" modification, are permitted provided that the following conditions
.\" are met:
.\" 1. Redistributions of source code must retain the above copyright
.\" notice, this list of conditions and the following disclaimer.
.\" 2. Redistributions in binary form must reproduce the above copyright
.\" notice, this list of conditions and the following disclaimer in the
.\" documentation and/or other materials provided with the distribution.
.\"
.\" THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY EXPRESS OR
.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
.\" OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
.\" IN NO EVENT SHALL THE DEVELOPERS BE LIABLE FOR ANY DIRECT, INDIRECT,
.\" INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
.\" NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
.\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
.\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
.\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
.\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.\"
.\" $FreeBSD$
.\"
.Dd June 19, 2019
.Dt SLEEPQUEUE 9
.Os
.Sh NAME
.Nm init_sleepqueues ,
.Nm sleepq_abort ,
.Nm sleepq_add ,
.Nm sleepq_alloc ,
.Nm sleepq_broadcast ,
.Nm sleepq_free ,
.Nm sleepq_lock ,
.Nm sleepq_lookup ,
.Nm sleepq_release ,
.Nm sleepq_remove ,
.Nm sleepq_signal ,
.Nm sleepq_set_timeout ,
.Nm sleepq_set_timeout_sbt ,
.Nm sleepq_sleepcnt ,
.Nm sleepq_timedwait ,
.Nm sleepq_timedwait_sig ,
.Nm sleepq_type ,
.Nm sleepq_wait ,
.Nm sleepq_wait_sig
.Nd manage the queues of sleeping threads
.Sh SYNOPSIS
.In sys/param.h
.In sys/sleepqueue.h
.Ft void
.Fn init_sleepqueues "void"
.Ft int
.Fn sleepq_abort "struct thread *td"
.Ft void
-.Fn sleepq_add "void *wchan" "struct lock_object *lock" "const char *wmesg" "int flags" "int queue"
+.Fn sleepq_add "const void *wchan" "struct lock_object *lock" "const char *wmesg" "int flags" "int queue"
.Ft struct sleepqueue *
.Fn sleepq_alloc "void"
.Ft int
-.Fn sleepq_broadcast "void *wchan" "int flags" "int pri" "int queue"
+.Fn sleepq_broadcast "const void *wchan" "int flags" "int pri" "int queue"
.Ft void
.Fn sleepq_free "struct sleepqueue *sq"
.Ft struct sleepqueue *
-.Fn sleepq_lookup "void *wchan"
+.Fn sleepq_lookup "const void *wchan"
.Ft void
-.Fn sleepq_lock "void *wchan"
+.Fn sleepq_lock "const void *wchan"
.Ft void
-.Fn sleepq_release "void *wchan"
+.Fn sleepq_release "const void *wchan"
.Ft void
-.Fn sleepq_remove "struct thread *td" "void *wchan"
+.Fn sleepq_remove "struct thread *td" "const void *wchan"
.Ft int
-.Fn sleepq_signal "void *wchan" "int flags" "int pri" "int queue"
+.Fn sleepq_signal "const void *wchan" "int flags" "int pri" "int queue"
.Ft void
-.Fn sleepq_set_timeout "void *wchan" "int timo"
+.Fn sleepq_set_timeout "const void *wchan" "int timo"
.Ft void
-.Fn sleepq_set_timeout_sbt "void *wchan" "sbintime_t sbt" \
+.Fn sleepq_set_timeout_sbt "const void *wchan" "sbintime_t sbt" \
"sbintime_t pr" "int flags"
.Ft u_int
-.Fn sleepq_sleepcnt "void *wchan" "int queue"
+.Fn sleepq_sleepcnt "const void *wchan" "int queue"
.Ft int
-.Fn sleepq_timedwait "void *wchan" "int pri"
+.Fn sleepq_timedwait "const void *wchan" "int pri"
.Ft int
-.Fn sleepq_timedwait_sig "void *wchan" "int pri"
+.Fn sleepq_timedwait_sig "const void *wchan" "int pri"
.Ft int
-.Fn sleepq_type "void *wchan"
+.Fn sleepq_type "const void *wchan"
.Ft void
-.Fn sleepq_wait "void *wchan" "int pri"
+.Fn sleepq_wait "const void *wchan" "int pri"
.Ft int
-.Fn sleepq_wait_sig "void *wchan" "int pri"
+.Fn sleepq_wait_sig "const void *wchan" "int pri"
.Sh DESCRIPTION
Sleep queues provide a mechanism for suspending execution of a thread until
some condition is met.
Each queue is associated with a specific wait channel when it is active,
and only one queue may be associated with a wait channel at any given point
in time.
The implementation of each wait channel splits its sleepqueue into 2 sub-queues
in order to enable some optimizations on threads' wakeups.
An active queue holds a list of threads that are blocked on the associated
wait channel.
Threads that are not blocked on a wait channel have an associated inactive
sleep queue.
When a thread blocks on a wait channel it donates its inactive sleep queue
to the wait channel.
When a thread is resumed,
the wait channel that it was blocked on gives it an inactive sleep queue for
later use.
.Pp
The
.Fn sleepq_alloc
function allocates an inactive sleep queue and is used to assign a
sleep queue to a thread during thread creation.
The
.Fn sleepq_free
function frees the resources associated with an inactive sleep queue and is
used to free a queue during thread destruction.
.Pp
Active sleep queues are stored in a hash table hashed on the addresses pointed
to by wait channels.
Each bucket in the hash table contains a sleep queue chain.
A sleep queue chain contains a spin mutex and a list of sleep queues that hash
to that specific chain.
Active sleep queues are protected by their chain's spin mutex.
The
.Fn init_sleepqueues
function initializes the hash table of sleep queue chains.
.Pp
The
.Fn sleepq_lock
function locks the sleep queue chain associated with wait channel
.Fa wchan .
.Pp
The
.Fn sleepq_lookup
returns a pointer to the currently active sleep queue for that wait
channel associated with
.Fa wchan
or
.Dv NULL
if there is no active sleep queue associated with
argument
.Fa wchan .
It requires the sleep queue chain associated with
.Fa wchan
to have been locked by a prior call to
.Fn sleepq_lock .
.Pp
The
.Fn sleepq_release
function unlocks the sleep queue chain associated with
.Fn wchan
and is primarily useful when aborting a pending sleep request before one of
the wait functions is called.
.Pp
The
.Fn sleepq_add
function places the current thread on the sleep queue associated with the
wait channel
.Fa wchan .
The sleep queue chain associated with argument
.Fa wchan
must be locked by a prior call to
.Fn sleepq_lock
when this function is called.
If a lock is specified via the
.Fa lock
argument, and if the kernel was compiled with
.Cd "options INVARIANTS" ,
then the sleep queue code will perform extra checks to ensure that
the lock is used by all threads sleeping on
.Fa wchan .
The
.Fa wmesg
parameter should be a short description of
.Fa wchan .
The
.Fa flags
parameter is a bitmask consisting of the type of sleep queue being slept on
and zero or more optional flags.
The
.Fa queue
parameter specifies the sub-queue, in which the contending thread will be
inserted.
.Pp
There are currently three types of sleep queues:
.Pp
.Bl -tag -width ".Dv SLEEPQ_CONDVAR" -compact
.It Dv SLEEPQ_CONDVAR
A sleep queue used to implement condition variables.
.It Dv SLEEPQ_SLEEP
A sleep queue used to implement
.Xr sleep 9 ,
.Xr wakeup 9
and
.Xr wakeup_one 9 .
.It Dv SLEEPQ_PAUSE
A sleep queue used to implement
.Xr pause 9 .
.El
.Pp
There are currently two optional flag:
.Pp
.Bl -tag -width ".Dv SLEEPQ_INTERRUPTIBLE" -compact
.It Dv SLEEPQ_INTERRUPTIBLE
The current thread is entering an interruptible sleep.
.El
.Bl -tag -width ".Dv SLEEPQ_STOP_ON_BDRY" -compact
.It Dv SLEEPQ_STOP_ON_BDRY
When thread is entering an interruptible sleep, do not stop it upon
arrival of stop action, like
.Dv SIGSTOP .
Wake it up instead.
.El
.Pp
A timeout on the sleep may be specified by calling
.Fn sleepq_set_timeout
after
.Fn sleepq_add .
The
.Fa wchan
parameter should be the same value from the preceding call to
.Fn sleepq_add ,
and the sleep queue chain associated with
.Fa wchan
must have been locked by a prior call to
.Fn sleepq_lock .
The
.Fa timo
parameter should specify the timeout value in ticks.
.Pp
.Fn sleepq_set_timeout_sbt
function takes
.Fa sbt
argument instead of
.Fa timo .
It allows to specify relative or absolute wakeup time with higher resolution
in form of
.Vt sbintime_t .
The parameter
.Fa pr
allows to specify wanted absolute event precision.
The parameter
.Fa flags
allows to pass additional
.Fn callout_reset_sbt
flags.
.Pp
Once the thread is ready to suspend,
one of the wait functions is called to put the current thread to sleep
until it is awakened and to context switch to another thread.
The
.Fn sleepq_wait
function is used for non-interruptible sleeps that do not have a timeout.
The
.Fn sleepq_timedwait
function is used for non-interruptible sleeps that have had a timeout set via
.Fn sleepq_set_timeout .
The
.Fn sleepq_wait_sig
function is used for interruptible sleeps that do not have a timeout.
The
.Fn sleepq_timedwait_sig
function is used for interruptible sleeps that do have a timeout set.
The
.Fa wchan
argument to all of the wait functions is the wait channel being slept
on.
The sleep queue chain associated with argument
.Fa wchan
needs to have been locked with a prior call to
.Fn sleepq_lock .
The
.Fa pri
argument is used to set the priority of the thread when it is awakened.
If it is set to zero, the thread's priority is left alone.
.Pp
When the thread is resumed,
the wait functions return a non-zero value if the thread was awakened due to
an interrupt other than a signal or a timeout.
If the sleep timed out, then
.Er EWOULDBLOCK
is returned.
If the sleep was interrupted by something other than a signal,
then some other return value will be returned.
.Pp
A sleeping thread is normally resumed by the
.Fn sleepq_broadcast
and
.Fn sleepq_signal
functions.
The
.Fn sleepq_signal
function awakens the highest priority thread sleeping on a wait channel
(if SLEEPQ_UNFAIR flag is set, thread that went to sleep recently) while
.Fn sleepq_broadcast
awakens all of the threads sleeping on a wait channel.
The
.Fa wchan
argument specifics which wait channel to awaken.
The
.Fa flags
argument must match the sleep queue type contained in the
.Fa flags
argument passed to
.Fn sleepq_add
by the threads sleeping on the wait channel.
If the
.Fa pri
argument does not equal \-1,
then each thread that is awakened will have its priority raised to
.Fa pri
if it has a lower priority.
The sleep queue chain associated with argument
.Fa wchan
must be locked by a prior call to
.Fn sleepq_lock
before calling any of these functions.
The
.Fa queue
argument specifies the sub-queue, from which threads need to be woken up.
.Pp
A thread in an interruptible sleep can be interrupted by another thread via
the
.Fn sleepq_abort
function.
The
.Fa td
argument specifies the thread to interrupt.
An individual thread can also be awakened from sleeping on a specific wait
channel via the
.Fn sleepq_remove
function.
The
.Fa td
argument specifies the thread to awaken and the
.Fa wchan
argument specifies the wait channel to awaken it from.
If the thread
.Fa td
is not blocked on the wait channel
.Fa wchan
then this function will not do anything,
even if the thread is asleep on a different wait channel.
This function should only be used if one of the other functions above is not
sufficient.
One possible use is waking up a specific thread from a widely shared sleep
channel.
.Pp
The
.Fn sleepq_sleepcnt
function offer a simple way to retrieve the number of threads sleeping for
the specified
.Fa queue ,
given a
.Fa wchan .
.Pp
The
.Fn sleepq_type
function returns the type of
.Fa wchan
associated to a sleepqueue.
.Pp
The
.Fn sleepq_abort ,
.Fn sleepq_broadcast ,
and
.Fn sleepq_signal
functions all return a boolean value.
If the return value is true,
then at least one thread was resumed that is currently swapped out.
The caller is responsible for awakening the scheduler process so that the
resumed thread will be swapped back in.
This is done by calling the
.Fn kick_proc0
function after releasing the sleep queue chain lock via a call to
.Fn sleepq_release .
.Pp
The sleep queue interface is currently used to implement the
.Xr sleep 9
and
.Xr condvar 9
interfaces.
Almost all other code in the kernel should use one of those interfaces rather
than manipulating sleep queues directly.
.Sh SEE ALSO
.Xr condvar 9 ,
.Xr runqueue 9 ,
.Xr scheduler 9 ,
.Xr sleep 9 ,
.Xr timeout 9
Index: head/sys/ddb/db_ps.c
===================================================================
--- head/sys/ddb/db_ps.c (revision 356056)
+++ head/sys/ddb/db_ps.c (revision 356057)
@@ -1,537 +1,537 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1993 The Regents of the University of California.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_kstack_pages.h"
#include <sys/param.h>
#include <sys/cons.h>
#include <sys/jail.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/proc.h>
#include <sys/sysent.h>
#include <sys/systm.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <ddb/ddb.h>
#define PRINT_NONE 0
#define PRINT_ARGS 1
static void dumpthread(volatile struct proc *p, volatile struct thread *td,
int all);
static void db_ps_proc(struct proc *p);
static int ps_mode;
/*
* At least one non-optional show-command must be implemented using
* DB_SHOW_ALL_COMMAND() so that db_show_all_cmd_set gets created.
* Here is one.
*/
DB_SHOW_ALL_COMMAND(procs, db_procs_cmd)
{
db_ps(addr, have_addr, count, modif);
}
static void
dump_args(volatile struct proc *p)
{
char *args;
int i, len;
if (p->p_args == NULL)
return;
args = p->p_args->ar_args;
len = (int)p->p_args->ar_length;
for (i = 0; i < len; i++) {
if (args[i] == '\0')
db_printf(" ");
else
db_printf("%c", args[i]);
}
}
/*
* Layout:
* - column counts
* - header
* - single-threaded process
* - multi-threaded process
* - thread in a MT process
*
* 1 2 3 4 5 6 7
* 1234567890123456789012345678901234567890123456789012345678901234567890
* pid ppid pgrp uid state wmesg wchan cmd
* <pid> <ppi> <pgi> <uid> <stat> <wmesg> <wchan > <name>
* <pid> <ppi> <pgi> <uid> <stat> (threaded) <command>
* <tid > <stat> <wmesg> <wchan > <name>
*
* For machines with 64-bit pointers, we expand the wchan field 8 more
* characters.
*/
void
db_ps(db_expr_t addr, bool hasaddr, db_expr_t count, char *modif)
{
struct proc *p;
int i, j;
ps_mode = modif[0] == 'a' ? PRINT_ARGS : PRINT_NONE;
#ifdef __LP64__
db_printf(" pid ppid pgrp uid state wmesg wchan cmd\n");
#else
db_printf(" pid ppid pgrp uid state wmesg wchan cmd\n");
#endif
if (!LIST_EMPTY(&allproc))
p = LIST_FIRST(&allproc);
else
p = &proc0;
for (; p != NULL && !db_pager_quit; p = LIST_NEXT(p, p_list))
db_ps_proc(p);
/*
* Do zombies.
*/
for (i = 0; i < pidhashlock + 1 && !db_pager_quit; i++) {
for (j = i; j <= pidhash && !db_pager_quit; j += pidhashlock + 1) {
LIST_FOREACH(p, &pidhashtbl[j], p_hash) {
if (p->p_state == PRS_ZOMBIE)
db_ps_proc(p);
}
}
}
}
static void
db_ps_proc(struct proc *p)
{
volatile struct proc *pp;
volatile struct thread *td;
struct ucred *cred;
struct pgrp *pgrp;
char state[9];
int rflag, sflag, dflag, lflag, wflag;
pp = p->p_pptr;
if (pp == NULL)
pp = p;
cred = p->p_ucred;
pgrp = p->p_pgrp;
db_printf("%5d %5d %5d %5d ", p->p_pid, pp->p_pid,
pgrp != NULL ? pgrp->pg_id : 0,
cred != NULL ? cred->cr_ruid : 0);
/* Determine our primary process state. */
switch (p->p_state) {
case PRS_NORMAL:
if (P_SHOULDSTOP(p))
state[0] = 'T';
else {
/*
* One of D, L, R, S, W. For a
* multithreaded process we will use
* the state of the thread with the
* highest precedence. The
* precendence order from high to low
* is R, L, D, S, W. If no thread is
* in a sane state we use '?' for our
* primary state.
*/
rflag = sflag = dflag = lflag = wflag = 0;
FOREACH_THREAD_IN_PROC(p, td) {
if (td->td_state == TDS_RUNNING ||
td->td_state == TDS_RUNQ ||
td->td_state == TDS_CAN_RUN)
rflag++;
if (TD_ON_LOCK(td))
lflag++;
if (TD_IS_SLEEPING(td)) {
if (!(td->td_flags & TDF_SINTR))
dflag++;
else
sflag++;
}
if (TD_AWAITING_INTR(td))
wflag++;
}
if (rflag)
state[0] = 'R';
else if (lflag)
state[0] = 'L';
else if (dflag)
state[0] = 'D';
else if (sflag)
state[0] = 'S';
else if (wflag)
state[0] = 'W';
else
state[0] = '?';
}
break;
case PRS_NEW:
state[0] = 'N';
break;
case PRS_ZOMBIE:
state[0] = 'Z';
break;
default:
state[0] = 'U';
break;
}
state[1] = '\0';
/* Additional process state flags. */
if (!(p->p_flag & P_INMEM))
strlcat(state, "W", sizeof(state));
if (p->p_flag & P_TRACED)
strlcat(state, "X", sizeof(state));
if (p->p_flag & P_WEXIT && p->p_state != PRS_ZOMBIE)
strlcat(state, "E", sizeof(state));
if (p->p_flag & P_PPWAIT)
strlcat(state, "V", sizeof(state));
if (p->p_flag & P_SYSTEM || p->p_lock > 0)
strlcat(state, "L", sizeof(state));
if (p->p_pgrp != NULL && p->p_session != NULL &&
SESS_LEADER(p))
strlcat(state, "s", sizeof(state));
/* Cheated here and didn't compare pgid's. */
if (p->p_flag & P_CONTROLT)
strlcat(state, "+", sizeof(state));
if (cred != NULL && jailed(cred))
strlcat(state, "J", sizeof(state));
db_printf(" %-6.6s ", state);
if (p->p_flag & P_HADTHREADS) {
#ifdef __LP64__
db_printf(" (threaded) ");
#else
db_printf(" (threaded) ");
#endif
if (p->p_flag & P_SYSTEM)
db_printf("[");
db_printf("%s", p->p_comm);
if (p->p_flag & P_SYSTEM)
db_printf("]");
if (ps_mode == PRINT_ARGS) {
db_printf(" ");
dump_args(p);
}
db_printf("\n");
}
FOREACH_THREAD_IN_PROC(p, td) {
dumpthread(p, td, p->p_flag & P_HADTHREADS);
if (db_pager_quit)
break;
}
}
static void
dumpthread(volatile struct proc *p, volatile struct thread *td, int all)
{
char state[9], wprefix;
const char *wmesg;
- void *wchan;
+ const void *wchan;
if (all) {
db_printf("%6d ", td->td_tid);
switch (td->td_state) {
case TDS_RUNNING:
snprintf(state, sizeof(state), "Run");
break;
case TDS_RUNQ:
snprintf(state, sizeof(state), "RunQ");
break;
case TDS_CAN_RUN:
snprintf(state, sizeof(state), "CanRun");
break;
case TDS_INACTIVE:
snprintf(state, sizeof(state), "Inactv");
break;
case TDS_INHIBITED:
state[0] = '\0';
if (TD_ON_LOCK(td))
strlcat(state, "L", sizeof(state));
if (TD_IS_SLEEPING(td)) {
if (td->td_flags & TDF_SINTR)
strlcat(state, "S", sizeof(state));
else
strlcat(state, "D", sizeof(state));
}
if (TD_IS_SWAPPED(td))
strlcat(state, "W", sizeof(state));
if (TD_AWAITING_INTR(td))
strlcat(state, "I", sizeof(state));
if (TD_IS_SUSPENDED(td))
strlcat(state, "s", sizeof(state));
if (state[0] != '\0')
break;
default:
snprintf(state, sizeof(state), "???");
}
db_printf(" %-6.6s ", state);
}
wprefix = ' ';
if (TD_ON_LOCK(td)) {
wprefix = '*';
wmesg = td->td_lockname;
wchan = td->td_blocked;
} else if (TD_ON_SLEEPQ(td)) {
wmesg = td->td_wmesg;
wchan = td->td_wchan;
} else if (TD_IS_RUNNING(td)) {
snprintf(state, sizeof(state), "CPU %d", td->td_oncpu);
wmesg = state;
wchan = NULL;
} else {
wmesg = "";
wchan = NULL;
}
db_printf("%c%-7.7s ", wprefix, wmesg);
if (wchan == NULL)
#ifdef __LP64__
db_printf("%18s ", "");
#else
db_printf("%10s ", "");
#endif
else
db_printf("%p ", wchan);
if (p->p_flag & P_SYSTEM)
db_printf("[");
if (td->td_name[0] != '\0')
db_printf("%s", td->td_name);
else
db_printf("%s", td->td_proc->p_comm);
if (p->p_flag & P_SYSTEM)
db_printf("]");
if (ps_mode == PRINT_ARGS && all == 0) {
db_printf(" ");
dump_args(p);
}
db_printf("\n");
}
DB_SHOW_COMMAND(thread, db_show_thread)
{
struct thread *td;
struct lock_object *lock;
u_int delta;
bool comma;
/* Determine which thread to examine. */
if (have_addr)
td = db_lookup_thread(addr, false);
else
td = kdb_thread;
lock = (struct lock_object *)td->td_lock;
db_printf("Thread %d at %p:\n", td->td_tid, td);
db_printf(" proc (pid %d): %p\n", td->td_proc->p_pid, td->td_proc);
if (td->td_name[0] != '\0')
db_printf(" name: %s\n", td->td_name);
db_printf(" pcb: %p\n", td->td_pcb);
db_printf(" stack: %p-%p\n", (void *)td->td_kstack,
(void *)(td->td_kstack + td->td_kstack_pages * PAGE_SIZE - 1));
db_printf(" flags: %#x ", td->td_flags);
db_printf(" pflags: %#x\n", td->td_pflags);
db_printf(" state: ");
switch (td->td_state) {
case TDS_INACTIVE:
db_printf("INACTIVE\n");
break;
case TDS_CAN_RUN:
db_printf("CAN RUN\n");
break;
case TDS_RUNQ:
db_printf("RUNQ\n");
break;
case TDS_RUNNING:
db_printf("RUNNING (CPU %d)\n", td->td_oncpu);
break;
case TDS_INHIBITED:
db_printf("INHIBITED: {");
comma = false;
if (TD_IS_SLEEPING(td)) {
db_printf("SLEEPING");
comma = true;
}
if (TD_IS_SUSPENDED(td)) {
if (comma)
db_printf(", ");
db_printf("SUSPENDED");
comma = true;
}
if (TD_IS_SWAPPED(td)) {
if (comma)
db_printf(", ");
db_printf("SWAPPED");
comma = true;
}
if (TD_ON_LOCK(td)) {
if (comma)
db_printf(", ");
db_printf("LOCK");
comma = true;
}
if (TD_AWAITING_INTR(td)) {
if (comma)
db_printf(", ");
db_printf("IWAIT");
}
db_printf("}\n");
break;
default:
db_printf("??? (%#x)\n", td->td_state);
break;
}
if (TD_ON_LOCK(td))
db_printf(" lock: %s turnstile: %p\n", td->td_lockname,
td->td_blocked);
if (TD_ON_SLEEPQ(td))
db_printf(
" wmesg: %s wchan: %p sleeptimo %lx. %jx (curr %lx. %jx)\n",
td->td_wmesg, td->td_wchan,
(long)sbttobt(td->td_sleeptimo).sec,
(uintmax_t)sbttobt(td->td_sleeptimo).frac,
(long)sbttobt(sbinuptime()).sec,
(uintmax_t)sbttobt(sbinuptime()).frac);
db_printf(" priority: %d\n", td->td_priority);
db_printf(" container lock: %s (%p)\n", lock->lo_name, lock);
if (td->td_swvoltick != 0) {
delta = ticks - td->td_swvoltick;
db_printf(" last voluntary switch: %u.%03u s ago\n",
delta / hz, (delta % hz) * 1000 / hz);
}
if (td->td_swinvoltick != 0) {
delta = ticks - td->td_swinvoltick;
db_printf(" last involuntary switch: %u.%03u s ago\n",
delta / hz, (delta % hz) * 1000 / hz);
}
}
DB_SHOW_COMMAND(proc, db_show_proc)
{
struct thread *td;
struct proc *p;
int i;
/* Determine which process to examine. */
if (have_addr)
p = db_lookup_proc(addr);
else
p = kdb_thread->td_proc;
db_printf("Process %d (%s) at %p:\n", p->p_pid, p->p_comm, p);
db_printf(" state: ");
switch (p->p_state) {
case PRS_NEW:
db_printf("NEW\n");
break;
case PRS_NORMAL:
db_printf("NORMAL\n");
break;
case PRS_ZOMBIE:
db_printf("ZOMBIE\n");
break;
default:
db_printf("??? (%#x)\n", p->p_state);
}
if (p->p_ucred != NULL) {
db_printf(" uid: %d gids: ", p->p_ucred->cr_uid);
for (i = 0; i < p->p_ucred->cr_ngroups; i++) {
db_printf("%d", p->p_ucred->cr_groups[i]);
if (i < (p->p_ucred->cr_ngroups - 1))
db_printf(", ");
}
db_printf("\n");
}
if (p->p_pptr != NULL)
db_printf(" parent: pid %d at %p\n", p->p_pptr->p_pid,
p->p_pptr);
if (p->p_leader != NULL && p->p_leader != p)
db_printf(" leader: pid %d at %p\n", p->p_leader->p_pid,
p->p_leader);
if (p->p_sysent != NULL)
db_printf(" ABI: %s\n", p->p_sysent->sv_name);
if (p->p_args != NULL) {
db_printf(" arguments: ");
dump_args(p);
db_printf("\n");
}
db_printf(" reaper: %p reapsubtree: %d\n",
p->p_reaper, p->p_reapsubtree);
db_printf(" sigparent: %d\n", p->p_sigparent);
db_printf(" vmspace: %p\n", p->p_vmspace);
db_printf(" (map %p)\n",
(p->p_vmspace != NULL) ? &p->p_vmspace->vm_map : 0);
db_printf(" (map.pmap %p)\n",
(p->p_vmspace != NULL) ? &p->p_vmspace->vm_map.pmap : 0);
db_printf(" (pmap %p)\n",
(p->p_vmspace != NULL) ? &p->p_vmspace->vm_pmap : 0);
db_printf(" threads: %d\n", p->p_numthreads);
FOREACH_THREAD_IN_PROC(p, td) {
dumpthread(p, td, 1);
if (db_pager_quit)
break;
}
}
void
db_findstack_cmd(db_expr_t addr, bool have_addr, db_expr_t dummy3 __unused,
char *dummy4 __unused)
{
struct proc *p;
struct thread *td;
vm_offset_t saddr;
if (have_addr)
saddr = addr;
else {
db_printf("Usage: findstack <address>\n");
return;
}
FOREACH_PROC_IN_SYSTEM(p) {
FOREACH_THREAD_IN_PROC(p, td) {
if (td->td_kstack <= saddr && saddr < td->td_kstack +
PAGE_SIZE * td->td_kstack_pages) {
db_printf("Thread %p\n", td);
return;
}
}
}
}
Index: head/sys/kern/kern_clock.c
===================================================================
--- head/sys/kern/kern_clock.c (revision 356056)
+++ head/sys/kern/kern_clock.c (revision 356057)
@@ -1,831 +1,831 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1982, 1986, 1991, 1993
* The Regents of the University of California. All rights reserved.
* (c) UNIX System Laboratories, Inc.
* All or some portions of this file are derived from material licensed
* to the University of California by American Telephone and Telegraph
* Co. or Unix System Laboratories, Inc. and are reproduced herein with
* the permission of UNIX System Laboratories, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)kern_clock.c 8.5 (Berkeley) 1/21/94
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_kdb.h"
#include "opt_device_polling.h"
#include "opt_hwpmc_hooks.h"
#include "opt_ntp.h"
#include "opt_watchdog.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/callout.h>
#include <sys/epoch.h>
#include <sys/eventhandler.h>
#include <sys/gtaskqueue.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/kthread.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/resource.h>
#include <sys/resourcevar.h>
#include <sys/sched.h>
#include <sys/sdt.h>
#include <sys/signalvar.h>
#include <sys/sleepqueue.h>
#include <sys/smp.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <sys/sysctl.h>
#include <sys/bus.h>
#include <sys/interrupt.h>
#include <sys/limits.h>
#include <sys/timetc.h>
#ifdef GPROF
#include <sys/gmon.h>
#endif
#ifdef HWPMC_HOOKS
#include <sys/pmckern.h>
PMC_SOFT_DEFINE( , , clock, hard);
PMC_SOFT_DEFINE( , , clock, stat);
PMC_SOFT_DEFINE_EX( , , clock, prof, \
cpu_startprofclock, cpu_stopprofclock);
#endif
#ifdef DEVICE_POLLING
extern void hardclock_device_poll(void);
#endif /* DEVICE_POLLING */
static void initclocks(void *dummy);
SYSINIT(clocks, SI_SUB_CLOCKS, SI_ORDER_FIRST, initclocks, NULL);
/* Spin-lock protecting profiling statistics. */
static struct mtx time_lock;
SDT_PROVIDER_DECLARE(sched);
SDT_PROBE_DEFINE2(sched, , , tick, "struct thread *", "struct proc *");
static int
sysctl_kern_cp_time(SYSCTL_HANDLER_ARGS)
{
int error;
long cp_time[CPUSTATES];
#ifdef SCTL_MASK32
int i;
unsigned int cp_time32[CPUSTATES];
#endif
read_cpu_time(cp_time);
#ifdef SCTL_MASK32
if (req->flags & SCTL_MASK32) {
if (!req->oldptr)
return SYSCTL_OUT(req, 0, sizeof(cp_time32));
for (i = 0; i < CPUSTATES; i++)
cp_time32[i] = (unsigned int)cp_time[i];
error = SYSCTL_OUT(req, cp_time32, sizeof(cp_time32));
} else
#endif
{
if (!req->oldptr)
return SYSCTL_OUT(req, 0, sizeof(cp_time));
error = SYSCTL_OUT(req, cp_time, sizeof(cp_time));
}
return error;
}
SYSCTL_PROC(_kern, OID_AUTO, cp_time, CTLTYPE_LONG|CTLFLAG_RD|CTLFLAG_MPSAFE,
0,0, sysctl_kern_cp_time, "LU", "CPU time statistics");
static long empty[CPUSTATES];
static int
sysctl_kern_cp_times(SYSCTL_HANDLER_ARGS)
{
struct pcpu *pcpu;
int error;
int c;
long *cp_time;
#ifdef SCTL_MASK32
unsigned int cp_time32[CPUSTATES];
int i;
#endif
if (!req->oldptr) {
#ifdef SCTL_MASK32
if (req->flags & SCTL_MASK32)
return SYSCTL_OUT(req, 0, sizeof(cp_time32) * (mp_maxid + 1));
else
#endif
return SYSCTL_OUT(req, 0, sizeof(long) * CPUSTATES * (mp_maxid + 1));
}
for (error = 0, c = 0; error == 0 && c <= mp_maxid; c++) {
if (!CPU_ABSENT(c)) {
pcpu = pcpu_find(c);
cp_time = pcpu->pc_cp_time;
} else {
cp_time = empty;
}
#ifdef SCTL_MASK32
if (req->flags & SCTL_MASK32) {
for (i = 0; i < CPUSTATES; i++)
cp_time32[i] = (unsigned int)cp_time[i];
error = SYSCTL_OUT(req, cp_time32, sizeof(cp_time32));
} else
#endif
error = SYSCTL_OUT(req, cp_time, sizeof(long) * CPUSTATES);
}
return error;
}
SYSCTL_PROC(_kern, OID_AUTO, cp_times, CTLTYPE_LONG|CTLFLAG_RD|CTLFLAG_MPSAFE,
0,0, sysctl_kern_cp_times, "LU", "per-CPU time statistics");
#ifdef DEADLKRES
static const char *blessed[] = {
"getblk",
"so_snd_sx",
"so_rcv_sx",
NULL
};
static int slptime_threshold = 1800;
static int blktime_threshold = 900;
static int sleepfreq = 3;
static void
deadlres_td_on_lock(struct proc *p, struct thread *td, int blkticks)
{
int tticks;
sx_assert(&allproc_lock, SX_LOCKED);
PROC_LOCK_ASSERT(p, MA_OWNED);
THREAD_LOCK_ASSERT(td, MA_OWNED);
/*
* The thread should be blocked on a turnstile, simply check
* if the turnstile channel is in good state.
*/
MPASS(td->td_blocked != NULL);
tticks = ticks - td->td_blktick;
if (tticks > blkticks)
/*
* Accordingly with provided thresholds, this thread is stuck
* for too long on a turnstile.
*/
panic("%s: possible deadlock detected for %p, "
"blocked for %d ticks\n", __func__, td, tticks);
}
static void
deadlres_td_sleep_q(struct proc *p, struct thread *td, int slpticks)
{
- void *wchan;
+ const void *wchan;
int i, slptype, tticks;
sx_assert(&allproc_lock, SX_LOCKED);
PROC_LOCK_ASSERT(p, MA_OWNED);
THREAD_LOCK_ASSERT(td, MA_OWNED);
/*
* Check if the thread is sleeping on a lock, otherwise skip the check.
* Drop the thread lock in order to avoid a LOR with the sleepqueue
* spinlock.
*/
wchan = td->td_wchan;
tticks = ticks - td->td_slptick;
slptype = sleepq_type(wchan);
if ((slptype == SLEEPQ_SX || slptype == SLEEPQ_LK) &&
tticks > slpticks) {
/*
* Accordingly with provided thresholds, this thread is stuck
* for too long on a sleepqueue.
* However, being on a sleepqueue, we might still check for the
* blessed list.
*/
for (i = 0; blessed[i] != NULL; i++)
if (!strcmp(blessed[i], td->td_wmesg))
return;
panic("%s: possible deadlock detected for %p, "
"blocked for %d ticks\n", __func__, td, tticks);
}
}
static void
deadlkres(void)
{
struct proc *p;
struct thread *td;
int blkticks, slpticks, tryl;
tryl = 0;
for (;;) {
blkticks = blktime_threshold * hz;
slpticks = slptime_threshold * hz;
/*
* Avoid to sleep on the sx_lock in order to avoid a
* possible priority inversion problem leading to
* starvation.
* If the lock can't be held after 100 tries, panic.
*/
if (!sx_try_slock(&allproc_lock)) {
if (tryl > 100)
panic("%s: possible deadlock detected "
"on allproc_lock\n", __func__);
tryl++;
pause("allproc", sleepfreq * hz);
continue;
}
tryl = 0;
FOREACH_PROC_IN_SYSTEM(p) {
PROC_LOCK(p);
if (p->p_state == PRS_NEW) {
PROC_UNLOCK(p);
continue;
}
FOREACH_THREAD_IN_PROC(p, td) {
thread_lock(td);
if (TD_ON_LOCK(td))
deadlres_td_on_lock(p, td,
blkticks);
else if (TD_IS_SLEEPING(td))
deadlres_td_sleep_q(p, td,
slpticks);
thread_unlock(td);
}
PROC_UNLOCK(p);
}
sx_sunlock(&allproc_lock);
/* Sleep for sleepfreq seconds. */
pause("-", sleepfreq * hz);
}
}
static struct kthread_desc deadlkres_kd = {
"deadlkres",
deadlkres,
(struct thread **)NULL
};
SYSINIT(deadlkres, SI_SUB_CLOCKS, SI_ORDER_ANY, kthread_start, &deadlkres_kd);
static SYSCTL_NODE(_debug, OID_AUTO, deadlkres, CTLFLAG_RW, 0,
"Deadlock resolver");
SYSCTL_INT(_debug_deadlkres, OID_AUTO, slptime_threshold, CTLFLAG_RW,
&slptime_threshold, 0,
"Number of seconds within is valid to sleep on a sleepqueue");
SYSCTL_INT(_debug_deadlkres, OID_AUTO, blktime_threshold, CTLFLAG_RW,
&blktime_threshold, 0,
"Number of seconds within is valid to block on a turnstile");
SYSCTL_INT(_debug_deadlkres, OID_AUTO, sleepfreq, CTLFLAG_RW, &sleepfreq, 0,
"Number of seconds between any deadlock resolver thread run");
#endif /* DEADLKRES */
void
read_cpu_time(long *cp_time)
{
struct pcpu *pc;
int i, j;
/* Sum up global cp_time[]. */
bzero(cp_time, sizeof(long) * CPUSTATES);
CPU_FOREACH(i) {
pc = pcpu_find(i);
for (j = 0; j < CPUSTATES; j++)
cp_time[j] += pc->pc_cp_time[j];
}
}
#include <sys/watchdog.h>
static int watchdog_ticks;
static int watchdog_enabled;
static void watchdog_fire(void);
static void watchdog_config(void *, u_int, int *);
static void
watchdog_attach(void)
{
EVENTHANDLER_REGISTER(watchdog_list, watchdog_config, NULL, 0);
}
/*
* Clock handling routines.
*
* This code is written to operate with two timers that run independently of
* each other.
*
* The main timer, running hz times per second, is used to trigger interval
* timers, timeouts and rescheduling as needed.
*
* The second timer handles kernel and user profiling,
* and does resource use estimation. If the second timer is programmable,
* it is randomized to avoid aliasing between the two clocks. For example,
* the randomization prevents an adversary from always giving up the cpu
* just before its quantum expires. Otherwise, it would never accumulate
* cpu ticks. The mean frequency of the second timer is stathz.
*
* If no second timer exists, stathz will be zero; in this case we drive
* profiling and statistics off the main clock. This WILL NOT be accurate;
* do not do it unless absolutely necessary.
*
* The statistics clock may (or may not) be run at a higher rate while
* profiling. This profile clock runs at profhz. We require that profhz
* be an integral multiple of stathz.
*
* If the statistics clock is running fast, it must be divided by the ratio
* profhz/stathz for statistics. (For profiling, every tick counts.)
*
* Time-of-day is maintained using a "timecounter", which may or may
* not be related to the hardware generating the above mentioned
* interrupts.
*/
int stathz;
int profhz;
int profprocs;
volatile int ticks;
int psratio;
DPCPU_DEFINE_STATIC(int, pcputicks); /* Per-CPU version of ticks. */
#ifdef DEVICE_POLLING
static int devpoll_run = 0;
#endif
/*
* Initialize clock frequencies and start both clocks running.
*/
/* ARGSUSED*/
static void
initclocks(void *dummy)
{
int i;
/*
* Set divisors to 1 (normal case) and let the machine-specific
* code do its bit.
*/
mtx_init(&time_lock, "time lock", NULL, MTX_DEF);
cpu_initclocks();
/*
* Compute profhz/stathz, and fix profhz if needed.
*/
i = stathz ? stathz : hz;
if (profhz == 0)
profhz = i;
psratio = profhz / i;
#ifdef SW_WATCHDOG
/* Enable hardclock watchdog now, even if a hardware watchdog exists. */
watchdog_attach();
#else
/* Volunteer to run a software watchdog. */
if (wdog_software_attach == NULL)
wdog_software_attach = watchdog_attach;
#endif
}
static __noinline void
hardclock_itimer(struct thread *td, struct pstats *pstats, int cnt, int usermode)
{
struct proc *p;
int flags;
flags = 0;
p = td->td_proc;
if (usermode &&
timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value)) {
PROC_ITIMLOCK(p);
if (itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL],
tick * cnt) == 0)
flags |= TDF_ALRMPEND | TDF_ASTPENDING;
PROC_ITIMUNLOCK(p);
}
if (timevalisset(&pstats->p_timer[ITIMER_PROF].it_value)) {
PROC_ITIMLOCK(p);
if (itimerdecr(&pstats->p_timer[ITIMER_PROF],
tick * cnt) == 0)
flags |= TDF_PROFPEND | TDF_ASTPENDING;
PROC_ITIMUNLOCK(p);
}
if (flags != 0) {
thread_lock(td);
td->td_flags |= flags;
thread_unlock(td);
}
}
void
hardclock(int cnt, int usermode)
{
struct pstats *pstats;
struct thread *td = curthread;
struct proc *p = td->td_proc;
int *t = DPCPU_PTR(pcputicks);
int global, i, newticks;
/*
* Update per-CPU and possibly global ticks values.
*/
*t += cnt;
global = ticks;
do {
newticks = *t - global;
if (newticks <= 0) {
if (newticks < -1)
*t = global - 1;
newticks = 0;
break;
}
} while (!atomic_fcmpset_int(&ticks, &global, *t));
/*
* Run current process's virtual and profile time, as needed.
*/
pstats = p->p_stats;
if (__predict_false(
timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) ||
timevalisset(&pstats->p_timer[ITIMER_PROF].it_value)))
hardclock_itimer(td, pstats, cnt, usermode);
#ifdef HWPMC_HOOKS
if (PMC_CPU_HAS_SAMPLES(PCPU_GET(cpuid)))
PMC_CALL_HOOK_UNLOCKED(curthread, PMC_FN_DO_SAMPLES, NULL);
if (td->td_intr_frame != NULL)
PMC_SOFT_CALL_TF( , , clock, hard, td->td_intr_frame);
#endif
/* We are in charge to handle this tick duty. */
if (newticks > 0) {
tc_ticktock(newticks);
#ifdef DEVICE_POLLING
/* Dangerous and no need to call these things concurrently. */
if (atomic_cmpset_acq_int(&devpoll_run, 0, 1)) {
/* This is very short and quick. */
hardclock_device_poll();
atomic_store_rel_int(&devpoll_run, 0);
}
#endif /* DEVICE_POLLING */
if (watchdog_enabled > 0) {
i = atomic_fetchadd_int(&watchdog_ticks, -newticks);
if (i > 0 && i <= newticks)
watchdog_fire();
}
}
if (curcpu == CPU_FIRST())
cpu_tick_calibration();
if (__predict_false(DPCPU_GET(epoch_cb_count)))
GROUPTASK_ENQUEUE(DPCPU_PTR(epoch_cb_task));
}
void
hardclock_sync(int cpu)
{
int *t;
KASSERT(!CPU_ABSENT(cpu), ("Absent CPU %d", cpu));
t = DPCPU_ID_PTR(cpu, pcputicks);
*t = ticks;
}
/*
* Compute number of ticks in the specified amount of time.
*/
int
tvtohz(struct timeval *tv)
{
unsigned long ticks;
long sec, usec;
/*
* If the number of usecs in the whole seconds part of the time
* difference fits in a long, then the total number of usecs will
* fit in an unsigned long. Compute the total and convert it to
* ticks, rounding up and adding 1 to allow for the current tick
* to expire. Rounding also depends on unsigned long arithmetic
* to avoid overflow.
*
* Otherwise, if the number of ticks in the whole seconds part of
* the time difference fits in a long, then convert the parts to
* ticks separately and add, using similar rounding methods and
* overflow avoidance. This method would work in the previous
* case but it is slightly slower and assumes that hz is integral.
*
* Otherwise, round the time difference down to the maximum
* representable value.
*
* If ints have 32 bits, then the maximum value for any timeout in
* 10ms ticks is 248 days.
*/
sec = tv->tv_sec;
usec = tv->tv_usec;
if (usec < 0) {
sec--;
usec += 1000000;
}
if (sec < 0) {
#ifdef DIAGNOSTIC
if (usec > 0) {
sec++;
usec -= 1000000;
}
printf("tvotohz: negative time difference %ld sec %ld usec\n",
sec, usec);
#endif
ticks = 1;
} else if (sec <= LONG_MAX / 1000000)
ticks = howmany(sec * 1000000 + (unsigned long)usec, tick) + 1;
else if (sec <= LONG_MAX / hz)
ticks = sec * hz
+ howmany((unsigned long)usec, tick) + 1;
else
ticks = LONG_MAX;
if (ticks > INT_MAX)
ticks = INT_MAX;
return ((int)ticks);
}
/*
* Start profiling on a process.
*
* Kernel profiling passes proc0 which never exits and hence
* keeps the profile clock running constantly.
*/
void
startprofclock(struct proc *p)
{
PROC_LOCK_ASSERT(p, MA_OWNED);
if (p->p_flag & P_STOPPROF)
return;
if ((p->p_flag & P_PROFIL) == 0) {
p->p_flag |= P_PROFIL;
mtx_lock(&time_lock);
if (++profprocs == 1)
cpu_startprofclock();
mtx_unlock(&time_lock);
}
}
/*
* Stop profiling on a process.
*/
void
stopprofclock(struct proc *p)
{
PROC_LOCK_ASSERT(p, MA_OWNED);
if (p->p_flag & P_PROFIL) {
if (p->p_profthreads != 0) {
while (p->p_profthreads != 0) {
p->p_flag |= P_STOPPROF;
msleep(&p->p_profthreads, &p->p_mtx, PPAUSE,
"stopprof", 0);
}
}
if ((p->p_flag & P_PROFIL) == 0)
return;
p->p_flag &= ~P_PROFIL;
mtx_lock(&time_lock);
if (--profprocs == 0)
cpu_stopprofclock();
mtx_unlock(&time_lock);
}
}
/*
* Statistics clock. Updates rusage information and calls the scheduler
* to adjust priorities of the active thread.
*
* This should be called by all active processors.
*/
void
statclock(int cnt, int usermode)
{
struct rusage *ru;
struct vmspace *vm;
struct thread *td;
struct proc *p;
long rss;
long *cp_time;
uint64_t runtime, new_switchtime;
td = curthread;
p = td->td_proc;
cp_time = (long *)PCPU_PTR(cp_time);
if (usermode) {
/*
* Charge the time as appropriate.
*/
td->td_uticks += cnt;
if (p->p_nice > NZERO)
cp_time[CP_NICE] += cnt;
else
cp_time[CP_USER] += cnt;
} else {
/*
* Came from kernel mode, so we were:
* - handling an interrupt,
* - doing syscall or trap work on behalf of the current
* user process, or
* - spinning in the idle loop.
* Whichever it is, charge the time as appropriate.
* Note that we charge interrupts to the current process,
* regardless of whether they are ``for'' that process,
* so that we know how much of its real time was spent
* in ``non-process'' (i.e., interrupt) work.
*/
if ((td->td_pflags & TDP_ITHREAD) ||
td->td_intr_nesting_level >= 2) {
td->td_iticks += cnt;
cp_time[CP_INTR] += cnt;
} else {
td->td_pticks += cnt;
td->td_sticks += cnt;
if (!TD_IS_IDLETHREAD(td))
cp_time[CP_SYS] += cnt;
else
cp_time[CP_IDLE] += cnt;
}
}
/* Update resource usage integrals and maximums. */
MPASS(p->p_vmspace != NULL);
vm = p->p_vmspace;
ru = &td->td_ru;
ru->ru_ixrss += pgtok(vm->vm_tsize) * cnt;
ru->ru_idrss += pgtok(vm->vm_dsize) * cnt;
ru->ru_isrss += pgtok(vm->vm_ssize) * cnt;
rss = pgtok(vmspace_resident_count(vm));
if (ru->ru_maxrss < rss)
ru->ru_maxrss = rss;
KTR_POINT2(KTR_SCHED, "thread", sched_tdname(td), "statclock",
"prio:%d", td->td_priority, "stathz:%d", (stathz)?stathz:hz);
SDT_PROBE2(sched, , , tick, td, td->td_proc);
thread_lock_flags(td, MTX_QUIET);
/*
* Compute the amount of time during which the current
* thread was running, and add that to its total so far.
*/
new_switchtime = cpu_ticks();
runtime = new_switchtime - PCPU_GET(switchtime);
td->td_runtime += runtime;
td->td_incruntime += runtime;
PCPU_SET(switchtime, new_switchtime);
sched_clock(td, cnt);
thread_unlock(td);
#ifdef HWPMC_HOOKS
if (td->td_intr_frame != NULL)
PMC_SOFT_CALL_TF( , , clock, stat, td->td_intr_frame);
#endif
}
void
profclock(int cnt, int usermode, uintfptr_t pc)
{
struct thread *td;
#ifdef GPROF
struct gmonparam *g;
uintfptr_t i;
#endif
td = curthread;
if (usermode) {
/*
* Came from user mode; CPU was in user state.
* If this process is being profiled, record the tick.
* if there is no related user location yet, don't
* bother trying to count it.
*/
if (td->td_proc->p_flag & P_PROFIL)
addupc_intr(td, pc, cnt);
}
#ifdef GPROF
else {
/*
* Kernel statistics are just like addupc_intr, only easier.
*/
g = &_gmonparam;
if (g->state == GMON_PROF_ON && pc >= g->lowpc) {
i = PC_TO_I(g, pc);
if (i < g->textsize) {
KCOUNT(g, i) += cnt;
}
}
}
#endif
#ifdef HWPMC_HOOKS
if (td->td_intr_frame != NULL)
PMC_SOFT_CALL_TF( , , clock, prof, td->td_intr_frame);
#endif
}
/*
* Return information about system clocks.
*/
static int
sysctl_kern_clockrate(SYSCTL_HANDLER_ARGS)
{
struct clockinfo clkinfo;
/*
* Construct clockinfo structure.
*/
bzero(&clkinfo, sizeof(clkinfo));
clkinfo.hz = hz;
clkinfo.tick = tick;
clkinfo.profhz = profhz;
clkinfo.stathz = stathz ? stathz : hz;
return (sysctl_handle_opaque(oidp, &clkinfo, sizeof clkinfo, req));
}
SYSCTL_PROC(_kern, KERN_CLOCKRATE, clockrate,
CTLTYPE_STRUCT|CTLFLAG_RD|CTLFLAG_MPSAFE,
0, 0, sysctl_kern_clockrate, "S,clockinfo",
"Rate and period of various kernel clocks");
static void
watchdog_config(void *unused __unused, u_int cmd, int *error)
{
u_int u;
u = cmd & WD_INTERVAL;
if (u >= WD_TO_1SEC) {
watchdog_ticks = (1 << (u - WD_TO_1SEC)) * hz;
watchdog_enabled = 1;
*error = 0;
} else {
watchdog_enabled = 0;
}
}
/*
* Handle a watchdog timeout by dumping interrupt information and
* then either dropping to DDB or panicking.
*/
static void
watchdog_fire(void)
{
int nintr;
uint64_t inttotal;
u_long *curintr;
char *curname;
curintr = intrcnt;
curname = intrnames;
inttotal = 0;
nintr = sintrcnt / sizeof(u_long);
printf("interrupt total\n");
while (--nintr >= 0) {
if (*curintr)
printf("%-12s %20lu\n", curname, *curintr);
curname += strlen(curname) + 1;
inttotal += *curintr++;
}
printf("Total %20ju\n", (uintmax_t)inttotal);
#if defined(KDB) && !defined(KDB_UNATTENDED)
kdb_backtrace();
kdb_enter(KDB_WHY_WATCHDOG, "watchdog timeout");
#else
panic("watchdog timeout");
#endif
}
Index: head/sys/kern/kern_lock.c
===================================================================
--- head/sys/kern/kern_lock.c (revision 356056)
+++ head/sys/kern/kern_lock.c (revision 356057)
@@ -1,1797 +1,1797 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2008 Attilio Rao <attilio@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice(s), this list of conditions and the following disclaimer as
* the first lines of this file unmodified other than the possible
* addition of one or more copyright notices.
* 2. Redistributions in binary form must reproduce the above copyright
* notice(s), this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
* DAMAGE.
*/
#include "opt_ddb.h"
#include "opt_hwpmc_hooks.h"
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/kdb.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/lock_profile.h>
#include <sys/lockmgr.h>
#include <sys/lockstat.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/sleepqueue.h>
#ifdef DEBUG_LOCKS
#include <sys/stack.h>
#endif
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <machine/cpu.h>
#ifdef DDB
#include <ddb/ddb.h>
#endif
#ifdef HWPMC_HOOKS
#include <sys/pmckern.h>
PMC_SOFT_DECLARE( , , lock, failed);
#endif
CTASSERT(LK_UNLOCKED == (LK_UNLOCKED &
~(LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS)));
#define SQ_EXCLUSIVE_QUEUE 0
#define SQ_SHARED_QUEUE 1
#ifndef INVARIANTS
#define _lockmgr_assert(lk, what, file, line)
#endif
#define TD_SLOCKS_INC(td) ((td)->td_lk_slocks++)
#define TD_SLOCKS_DEC(td) ((td)->td_lk_slocks--)
#ifndef DEBUG_LOCKS
#define STACK_PRINT(lk)
#define STACK_SAVE(lk)
#define STACK_ZERO(lk)
#else
#define STACK_PRINT(lk) stack_print_ddb(&(lk)->lk_stack)
#define STACK_SAVE(lk) stack_save(&(lk)->lk_stack)
#define STACK_ZERO(lk) stack_zero(&(lk)->lk_stack)
#endif
#define LOCK_LOG2(lk, string, arg1, arg2) \
if (LOCK_LOG_TEST(&(lk)->lock_object, 0)) \
CTR2(KTR_LOCK, (string), (arg1), (arg2))
#define LOCK_LOG3(lk, string, arg1, arg2, arg3) \
if (LOCK_LOG_TEST(&(lk)->lock_object, 0)) \
CTR3(KTR_LOCK, (string), (arg1), (arg2), (arg3))
#define GIANT_DECLARE \
int _i = 0; \
WITNESS_SAVE_DECL(Giant)
#define GIANT_RESTORE() do { \
if (__predict_false(_i > 0)) { \
while (_i--) \
mtx_lock(&Giant); \
WITNESS_RESTORE(&Giant.lock_object, Giant); \
} \
} while (0)
#define GIANT_SAVE() do { \
if (__predict_false(mtx_owned(&Giant))) { \
WITNESS_SAVE(&Giant.lock_object, Giant); \
while (mtx_owned(&Giant)) { \
_i++; \
mtx_unlock(&Giant); \
} \
} \
} while (0)
static bool __always_inline
LK_CAN_SHARE(uintptr_t x, int flags, bool fp)
{
if ((x & (LK_SHARE | LK_EXCLUSIVE_WAITERS | LK_EXCLUSIVE_SPINNERS)) ==
LK_SHARE)
return (true);
if (fp || (!(x & LK_SHARE)))
return (false);
if ((curthread->td_lk_slocks != 0 && !(flags & LK_NODDLKTREAT)) ||
(curthread->td_pflags & TDP_DEADLKTREAT))
return (true);
return (false);
}
#define LK_TRYOP(x) \
((x) & LK_NOWAIT)
#define LK_CAN_WITNESS(x) \
(((x) & LK_NOWITNESS) == 0 && !LK_TRYOP(x))
#define LK_TRYWIT(x) \
(LK_TRYOP(x) ? LOP_TRYLOCK : 0)
#define lockmgr_disowned(lk) \
(((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == LK_KERNPROC)
#define lockmgr_xlocked_v(v) \
(((v) & ~(LK_FLAGMASK & ~LK_SHARE)) == (uintptr_t)curthread)
#define lockmgr_xlocked(lk) lockmgr_xlocked_v((lk)->lk_lock)
static void assert_lockmgr(const struct lock_object *lock, int how);
#ifdef DDB
static void db_show_lockmgr(const struct lock_object *lock);
#endif
static void lock_lockmgr(struct lock_object *lock, uintptr_t how);
#ifdef KDTRACE_HOOKS
static int owner_lockmgr(const struct lock_object *lock,
struct thread **owner);
#endif
static uintptr_t unlock_lockmgr(struct lock_object *lock);
struct lock_class lock_class_lockmgr = {
.lc_name = "lockmgr",
.lc_flags = LC_RECURSABLE | LC_SLEEPABLE | LC_SLEEPLOCK | LC_UPGRADABLE,
.lc_assert = assert_lockmgr,
#ifdef DDB
.lc_ddb_show = db_show_lockmgr,
#endif
.lc_lock = lock_lockmgr,
.lc_unlock = unlock_lockmgr,
#ifdef KDTRACE_HOOKS
.lc_owner = owner_lockmgr,
#endif
};
struct lockmgr_wait {
const char *iwmesg;
int ipri;
int itimo;
};
static bool __always_inline lockmgr_slock_try(struct lock *lk, uintptr_t *xp,
int flags, bool fp);
static bool __always_inline lockmgr_sunlock_try(struct lock *lk, uintptr_t *xp);
static void
lockmgr_exit(u_int flags, struct lock_object *ilk, int wakeup_swapper)
{
struct lock_class *class;
if (flags & LK_INTERLOCK) {
class = LOCK_CLASS(ilk);
class->lc_unlock(ilk);
}
if (__predict_false(wakeup_swapper))
kick_proc0();
}
static void
lockmgr_note_shared_acquire(struct lock *lk, int contested,
uint64_t waittime, const char *file, int line, int flags)
{
LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(lockmgr__acquire, lk, contested,
waittime, file, line, LOCKSTAT_READER);
LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file, line);
WITNESS_LOCK(&lk->lock_object, LK_TRYWIT(flags), file, line);
TD_LOCKS_INC(curthread);
TD_SLOCKS_INC(curthread);
STACK_SAVE(lk);
}
static void
lockmgr_note_shared_release(struct lock *lk, const char *file, int line)
{
LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk, LOCKSTAT_READER);
WITNESS_UNLOCK(&lk->lock_object, 0, file, line);
LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line);
TD_LOCKS_DEC(curthread);
TD_SLOCKS_DEC(curthread);
}
static void
lockmgr_note_exclusive_acquire(struct lock *lk, int contested,
uint64_t waittime, const char *file, int line, int flags)
{
LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(lockmgr__acquire, lk, contested,
waittime, file, line, LOCKSTAT_WRITER);
LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, lk->lk_recurse, file, line);
WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE | LK_TRYWIT(flags), file,
line);
TD_LOCKS_INC(curthread);
STACK_SAVE(lk);
}
static void
lockmgr_note_exclusive_release(struct lock *lk, const char *file, int line)
{
LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk, LOCKSTAT_WRITER);
LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file,
line);
WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
TD_LOCKS_DEC(curthread);
}
static __inline struct thread *
lockmgr_xholder(const struct lock *lk)
{
uintptr_t x;
x = lk->lk_lock;
return ((x & LK_SHARE) ? NULL : (struct thread *)LK_HOLDER(x));
}
/*
* It assumes sleepq_lock held and returns with this one unheld.
* It also assumes the generic interlock is sane and previously checked.
* If LK_INTERLOCK is specified the interlock is not reacquired after the
* sleep.
*/
static __inline int
sleeplk(struct lock *lk, u_int flags, struct lock_object *ilk,
const char *wmesg, int pri, int timo, int queue)
{
GIANT_DECLARE;
struct lock_class *class;
int catch, error;
class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
catch = pri & PCATCH;
pri &= PRIMASK;
error = 0;
LOCK_LOG3(lk, "%s: %p blocking on the %s sleepqueue", __func__, lk,
(queue == SQ_EXCLUSIVE_QUEUE) ? "exclusive" : "shared");
if (flags & LK_INTERLOCK)
class->lc_unlock(ilk);
if (queue == SQ_EXCLUSIVE_QUEUE && (flags & LK_SLEEPFAIL) != 0)
lk->lk_exslpfail++;
GIANT_SAVE();
sleepq_add(&lk->lock_object, NULL, wmesg, SLEEPQ_LK | (catch ?
SLEEPQ_INTERRUPTIBLE : 0), queue);
if ((flags & LK_TIMELOCK) && timo)
sleepq_set_timeout(&lk->lock_object, timo);
/*
* Decisional switch for real sleeping.
*/
if ((flags & LK_TIMELOCK) && timo && catch)
error = sleepq_timedwait_sig(&lk->lock_object, pri);
else if ((flags & LK_TIMELOCK) && timo)
error = sleepq_timedwait(&lk->lock_object, pri);
else if (catch)
error = sleepq_wait_sig(&lk->lock_object, pri);
else
sleepq_wait(&lk->lock_object, pri);
GIANT_RESTORE();
if ((flags & LK_SLEEPFAIL) && error == 0)
error = ENOLCK;
return (error);
}
static __inline int
wakeupshlk(struct lock *lk, const char *file, int line)
{
uintptr_t v, x, orig_x;
u_int realexslp;
int queue, wakeup_swapper;
wakeup_swapper = 0;
for (;;) {
x = lk->lk_lock;
if (lockmgr_sunlock_try(lk, &x))
break;
/*
* We should have a sharer with waiters, so enter the hard
* path in order to handle wakeups correctly.
*/
sleepq_lock(&lk->lock_object);
orig_x = lk->lk_lock;
retry_sleepq:
x = orig_x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
v = LK_UNLOCKED;
/*
* If the lock has exclusive waiters, give them preference in
* order to avoid deadlock with shared runners up.
* If interruptible sleeps left the exclusive queue empty
* avoid a starvation for the threads sleeping on the shared
* queue by giving them precedence and cleaning up the
* exclusive waiters bit anyway.
* Please note that lk_exslpfail count may be lying about
* the real number of waiters with the LK_SLEEPFAIL flag on
* because they may be used in conjunction with interruptible
* sleeps so lk_exslpfail might be considered an 'upper limit'
* bound, including the edge cases.
*/
realexslp = sleepq_sleepcnt(&lk->lock_object,
SQ_EXCLUSIVE_QUEUE);
if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) {
if (lk->lk_exslpfail < realexslp) {
lk->lk_exslpfail = 0;
queue = SQ_EXCLUSIVE_QUEUE;
v |= (x & LK_SHARED_WAITERS);
} else {
lk->lk_exslpfail = 0;
LOCK_LOG2(lk,
"%s: %p has only LK_SLEEPFAIL sleepers",
__func__, lk);
LOCK_LOG2(lk,
"%s: %p waking up threads on the exclusive queue",
__func__, lk);
wakeup_swapper =
sleepq_broadcast(&lk->lock_object,
SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE);
queue = SQ_SHARED_QUEUE;
}
} else {
/*
* Exclusive waiters sleeping with LK_SLEEPFAIL on
* and using interruptible sleeps/timeout may have
* left spourious lk_exslpfail counts on, so clean
* it up anyway.
*/
lk->lk_exslpfail = 0;
queue = SQ_SHARED_QUEUE;
}
if (lockmgr_sunlock_try(lk, &orig_x)) {
sleepq_release(&lk->lock_object);
break;
}
x |= LK_SHARERS_LOCK(1);
if (!atomic_fcmpset_rel_ptr(&lk->lk_lock, &x, v)) {
orig_x = x;
goto retry_sleepq;
}
LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
__func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
"exclusive");
wakeup_swapper |= sleepq_broadcast(&lk->lock_object, SLEEPQ_LK,
0, queue);
sleepq_release(&lk->lock_object);
break;
}
lockmgr_note_shared_release(lk, file, line);
return (wakeup_swapper);
}
static void
assert_lockmgr(const struct lock_object *lock, int what)
{
panic("lockmgr locks do not support assertions");
}
static void
lock_lockmgr(struct lock_object *lock, uintptr_t how)
{
panic("lockmgr locks do not support sleep interlocking");
}
static uintptr_t
unlock_lockmgr(struct lock_object *lock)
{
panic("lockmgr locks do not support sleep interlocking");
}
#ifdef KDTRACE_HOOKS
static int
owner_lockmgr(const struct lock_object *lock, struct thread **owner)
{
panic("lockmgr locks do not support owner inquiring");
}
#endif
void
lockinit(struct lock *lk, int pri, const char *wmesg, int timo, int flags)
{
int iflags;
MPASS((flags & ~LK_INIT_MASK) == 0);
ASSERT_ATOMIC_LOAD_PTR(lk->lk_lock,
("%s: lockmgr not aligned for %s: %p", __func__, wmesg,
&lk->lk_lock));
iflags = LO_SLEEPABLE | LO_UPGRADABLE;
if (flags & LK_CANRECURSE)
iflags |= LO_RECURSABLE;
if ((flags & LK_NODUP) == 0)
iflags |= LO_DUPOK;
if (flags & LK_NOPROFILE)
iflags |= LO_NOPROFILE;
if ((flags & LK_NOWITNESS) == 0)
iflags |= LO_WITNESS;
if (flags & LK_QUIET)
iflags |= LO_QUIET;
if (flags & LK_IS_VNODE)
iflags |= LO_IS_VNODE;
if (flags & LK_NEW)
iflags |= LO_NEW;
iflags |= flags & LK_NOSHARE;
lock_init(&lk->lock_object, &lock_class_lockmgr, wmesg, NULL, iflags);
lk->lk_lock = LK_UNLOCKED;
lk->lk_recurse = 0;
lk->lk_exslpfail = 0;
lk->lk_timo = timo;
lk->lk_pri = pri;
STACK_ZERO(lk);
}
/*
* XXX: Gross hacks to manipulate external lock flags after
* initialization. Used for certain vnode and buf locks.
*/
void
lockallowshare(struct lock *lk)
{
lockmgr_assert(lk, KA_XLOCKED);
lk->lock_object.lo_flags &= ~LK_NOSHARE;
}
void
lockdisableshare(struct lock *lk)
{
lockmgr_assert(lk, KA_XLOCKED);
lk->lock_object.lo_flags |= LK_NOSHARE;
}
void
lockallowrecurse(struct lock *lk)
{
lockmgr_assert(lk, KA_XLOCKED);
lk->lock_object.lo_flags |= LO_RECURSABLE;
}
void
lockdisablerecurse(struct lock *lk)
{
lockmgr_assert(lk, KA_XLOCKED);
lk->lock_object.lo_flags &= ~LO_RECURSABLE;
}
void
lockdestroy(struct lock *lk)
{
KASSERT(lk->lk_lock == LK_UNLOCKED, ("lockmgr still held"));
KASSERT(lk->lk_recurse == 0, ("lockmgr still recursed"));
KASSERT(lk->lk_exslpfail == 0, ("lockmgr still exclusive waiters"));
lock_destroy(&lk->lock_object);
}
static bool __always_inline
lockmgr_slock_try(struct lock *lk, uintptr_t *xp, int flags, bool fp)
{
/*
* If no other thread has an exclusive lock, or
* no exclusive waiter is present, bump the count of
* sharers. Since we have to preserve the state of
* waiters, if we fail to acquire the shared lock
* loop back and retry.
*/
*xp = lk->lk_lock;
while (LK_CAN_SHARE(*xp, flags, fp)) {
if (atomic_fcmpset_acq_ptr(&lk->lk_lock, xp,
*xp + LK_ONE_SHARER)) {
return (true);
}
}
return (false);
}
static bool __always_inline
lockmgr_sunlock_try(struct lock *lk, uintptr_t *xp)
{
for (;;) {
if (LK_SHARERS(*xp) > 1 || !(*xp & LK_ALL_WAITERS)) {
if (atomic_fcmpset_rel_ptr(&lk->lk_lock, xp,
*xp - LK_ONE_SHARER))
return (true);
continue;
}
break;
}
return (false);
}
static __noinline int
lockmgr_slock_hard(struct lock *lk, u_int flags, struct lock_object *ilk,
const char *file, int line, struct lockmgr_wait *lwa)
{
uintptr_t tid, x;
int error = 0;
const char *iwmesg;
int ipri, itimo;
#ifdef KDTRACE_HOOKS
uint64_t sleep_time = 0;
#endif
#ifdef LOCK_PROFILING
uint64_t waittime = 0;
int contested = 0;
#endif
if (__predict_false(panicstr != NULL))
goto out;
tid = (uintptr_t)curthread;
if (LK_CAN_WITNESS(flags))
WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
file, line, flags & LK_INTERLOCK ? ilk : NULL);
for (;;) {
if (lockmgr_slock_try(lk, &x, flags, false))
break;
#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);
#endif
lock_profile_obtain_lock_failed(&lk->lock_object,
&contested, &waittime);
/*
* If the lock is already held by curthread in
* exclusive way avoid a deadlock.
*/
if (LK_HOLDER(x) == tid) {
LOCK_LOG2(lk,
"%s: %p already held in exclusive mode",
__func__, lk);
error = EDEADLK;
break;
}
/*
* If the lock is expected to not sleep just give up
* and return.
*/
if (LK_TRYOP(flags)) {
LOCK_LOG2(lk, "%s: %p fails the try operation",
__func__, lk);
error = EBUSY;
break;
}
/*
* Acquire the sleepqueue chain lock because we
* probabilly will need to manipulate waiters flags.
*/
sleepq_lock(&lk->lock_object);
x = lk->lk_lock;
retry_sleepq:
/*
* if the lock can be acquired in shared mode, try
* again.
*/
if (LK_CAN_SHARE(x, flags, false)) {
sleepq_release(&lk->lock_object);
continue;
}
/*
* Try to set the LK_SHARED_WAITERS flag. If we fail,
* loop back and retry.
*/
if ((x & LK_SHARED_WAITERS) == 0) {
if (!atomic_fcmpset_acq_ptr(&lk->lk_lock, &x,
x | LK_SHARED_WAITERS)) {
goto retry_sleepq;
}
LOCK_LOG2(lk, "%s: %p set shared waiters flag",
__func__, lk);
}
if (lwa == NULL) {
iwmesg = lk->lock_object.lo_name;
ipri = lk->lk_pri;
itimo = lk->lk_timo;
} else {
iwmesg = lwa->iwmesg;
ipri = lwa->ipri;
itimo = lwa->itimo;
}
/*
* As far as we have been unable to acquire the
* shared lock and the shared waiters flag is set,
* we will sleep.
*/
#ifdef KDTRACE_HOOKS
sleep_time -= lockstat_nsecs(&lk->lock_object);
#endif
error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
SQ_SHARED_QUEUE);
#ifdef KDTRACE_HOOKS
sleep_time += lockstat_nsecs(&lk->lock_object);
#endif
flags &= ~LK_INTERLOCK;
if (error) {
LOCK_LOG3(lk,
"%s: interrupted sleep for %p with %d",
__func__, lk, error);
break;
}
LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
__func__, lk);
}
if (error == 0) {
#ifdef KDTRACE_HOOKS
if (sleep_time != 0)
LOCKSTAT_RECORD4(lockmgr__block, lk, sleep_time,
LOCKSTAT_READER, (x & LK_SHARE) == 0,
(x & LK_SHARE) == 0 ? 0 : LK_SHARERS(x));
#endif
#ifdef LOCK_PROFILING
lockmgr_note_shared_acquire(lk, contested, waittime,
file, line, flags);
#else
lockmgr_note_shared_acquire(lk, 0, 0, file, line,
flags);
#endif
}
out:
lockmgr_exit(flags, ilk, 0);
return (error);
}
static __noinline int
lockmgr_xlock_hard(struct lock *lk, u_int flags, struct lock_object *ilk,
const char *file, int line, struct lockmgr_wait *lwa)
{
struct lock_class *class;
uintptr_t tid, x, v;
int error = 0;
const char *iwmesg;
int ipri, itimo;
#ifdef KDTRACE_HOOKS
uint64_t sleep_time = 0;
#endif
#ifdef LOCK_PROFILING
uint64_t waittime = 0;
int contested = 0;
#endif
if (__predict_false(panicstr != NULL))
goto out;
tid = (uintptr_t)curthread;
if (LK_CAN_WITNESS(flags))
WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
ilk : NULL);
/*
* If curthread already holds the lock and this one is
* allowed to recurse, simply recurse on it.
*/
if (lockmgr_xlocked(lk)) {
if ((flags & LK_CANRECURSE) == 0 &&
(lk->lock_object.lo_flags & LO_RECURSABLE) == 0) {
/*
* If the lock is expected to not panic just
* give up and return.
*/
if (LK_TRYOP(flags)) {
LOCK_LOG2(lk,
"%s: %p fails the try operation",
__func__, lk);
error = EBUSY;
goto out;
}
if (flags & LK_INTERLOCK) {
class = LOCK_CLASS(ilk);
class->lc_unlock(ilk);
}
panic("%s: recursing on non recursive lockmgr %p "
"@ %s:%d\n", __func__, lk, file, line);
}
lk->lk_recurse++;
LOCK_LOG2(lk, "%s: %p recursing", __func__, lk);
LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0,
lk->lk_recurse, file, line);
WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
LK_TRYWIT(flags), file, line);
TD_LOCKS_INC(curthread);
goto out;
}
for (;;) {
if (lk->lk_lock == LK_UNLOCKED &&
atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid))
break;
#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);
#endif
lock_profile_obtain_lock_failed(&lk->lock_object,
&contested, &waittime);
/*
* If the lock is expected to not sleep just give up
* and return.
*/
if (LK_TRYOP(flags)) {
LOCK_LOG2(lk, "%s: %p fails the try operation",
__func__, lk);
error = EBUSY;
break;
}
/*
* Acquire the sleepqueue chain lock because we
* probabilly will need to manipulate waiters flags.
*/
sleepq_lock(&lk->lock_object);
x = lk->lk_lock;
retry_sleepq:
/*
* if the lock has been released while we spun on
* the sleepqueue chain lock just try again.
*/
if (x == LK_UNLOCKED) {
sleepq_release(&lk->lock_object);
continue;
}
/*
* The lock can be in the state where there is a
* pending queue of waiters, but still no owner.
* This happens when the lock is contested and an
* owner is going to claim the lock.
* If curthread is the one successfully acquiring it
* claim lock ownership and return, preserving waiters
* flags.
*/
v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
if ((x & ~v) == LK_UNLOCKED) {
v &= ~LK_EXCLUSIVE_SPINNERS;
if (atomic_fcmpset_acq_ptr(&lk->lk_lock, &x,
tid | v)) {
sleepq_release(&lk->lock_object);
LOCK_LOG2(lk,
"%s: %p claimed by a new writer",
__func__, lk);
break;
}
goto retry_sleepq;
}
/*
* Try to set the LK_EXCLUSIVE_WAITERS flag. If we
* fail, loop back and retry.
*/
if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
if (!atomic_fcmpset_ptr(&lk->lk_lock, &x,
x | LK_EXCLUSIVE_WAITERS)) {
goto retry_sleepq;
}
LOCK_LOG2(lk, "%s: %p set excl waiters flag",
__func__, lk);
}
if (lwa == NULL) {
iwmesg = lk->lock_object.lo_name;
ipri = lk->lk_pri;
itimo = lk->lk_timo;
} else {
iwmesg = lwa->iwmesg;
ipri = lwa->ipri;
itimo = lwa->itimo;
}
/*
* As far as we have been unable to acquire the
* exclusive lock and the exclusive waiters flag
* is set, we will sleep.
*/
#ifdef KDTRACE_HOOKS
sleep_time -= lockstat_nsecs(&lk->lock_object);
#endif
error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
SQ_EXCLUSIVE_QUEUE);
#ifdef KDTRACE_HOOKS
sleep_time += lockstat_nsecs(&lk->lock_object);
#endif
flags &= ~LK_INTERLOCK;
if (error) {
LOCK_LOG3(lk,
"%s: interrupted sleep for %p with %d",
__func__, lk, error);
break;
}
LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
__func__, lk);
}
if (error == 0) {
#ifdef KDTRACE_HOOKS
if (sleep_time != 0)
LOCKSTAT_RECORD4(lockmgr__block, lk, sleep_time,
LOCKSTAT_WRITER, (x & LK_SHARE) == 0,
(x & LK_SHARE) == 0 ? 0 : LK_SHARERS(x));
#endif
#ifdef LOCK_PROFILING
lockmgr_note_exclusive_acquire(lk, contested, waittime,
file, line, flags);
#else
lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
flags);
#endif
}
out:
lockmgr_exit(flags, ilk, 0);
return (error);
}
static __noinline int
lockmgr_upgrade(struct lock *lk, u_int flags, struct lock_object *ilk,
const char *file, int line, struct lockmgr_wait *lwa)
{
uintptr_t tid, x, v;
int error = 0;
int wakeup_swapper = 0;
int op;
if (__predict_false(panicstr != NULL))
goto out;
tid = (uintptr_t)curthread;
_lockmgr_assert(lk, KA_SLOCKED, file, line);
v = lk->lk_lock;
x = v & LK_ALL_WAITERS;
v &= LK_EXCLUSIVE_SPINNERS;
/*
* Try to switch from one shared lock to an exclusive one.
* We need to preserve waiters flags during the operation.
*/
if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x | v,
tid | x)) {
LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file,
line);
WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE |
LK_TRYWIT(flags), file, line);
LOCKSTAT_RECORD0(lockmgr__upgrade, lk);
TD_SLOCKS_DEC(curthread);
goto out;
}
op = flags & LK_TYPE_MASK;
/*
* In LK_TRYUPGRADE mode, do not drop the lock,
* returning EBUSY instead.
*/
if (op == LK_TRYUPGRADE) {
LOCK_LOG2(lk, "%s: %p failed the nowait upgrade",
__func__, lk);
error = EBUSY;
goto out;
}
/*
* We have been unable to succeed in upgrading, so just
* give up the shared lock.
*/
wakeup_swapper |= wakeupshlk(lk, file, line);
error = lockmgr_xlock_hard(lk, flags, ilk, file, line, lwa);
flags &= ~LK_INTERLOCK;
out:
lockmgr_exit(flags, ilk, wakeup_swapper);
return (error);
}
int
lockmgr_lock_fast_path(struct lock *lk, u_int flags, struct lock_object *ilk,
const char *file, int line)
{
struct lock_class *class;
uintptr_t x, tid;
u_int op;
bool locked;
if (__predict_false(panicstr != NULL))
return (0);
op = flags & LK_TYPE_MASK;
locked = false;
switch (op) {
case LK_SHARED:
if (LK_CAN_WITNESS(flags))
WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
file, line, flags & LK_INTERLOCK ? ilk : NULL);
if (__predict_false(lk->lock_object.lo_flags & LK_NOSHARE))
break;
if (lockmgr_slock_try(lk, &x, flags, true)) {
lockmgr_note_shared_acquire(lk, 0, 0,
file, line, flags);
locked = true;
} else {
return (lockmgr_slock_hard(lk, flags, ilk, file, line,
NULL));
}
break;
case LK_EXCLUSIVE:
if (LK_CAN_WITNESS(flags))
WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
ilk : NULL);
tid = (uintptr_t)curthread;
if (lk->lk_lock == LK_UNLOCKED &&
atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) {
lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
flags);
locked = true;
} else {
return (lockmgr_xlock_hard(lk, flags, ilk, file, line,
NULL));
}
break;
case LK_UPGRADE:
case LK_TRYUPGRADE:
return (lockmgr_upgrade(lk, flags, ilk, file, line, NULL));
default:
break;
}
if (__predict_true(locked)) {
if (__predict_false(flags & LK_INTERLOCK)) {
class = LOCK_CLASS(ilk);
class->lc_unlock(ilk);
}
return (0);
} else {
return (__lockmgr_args(lk, flags, ilk, LK_WMESG_DEFAULT,
LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, file, line));
}
}
static __noinline int
lockmgr_sunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk,
const char *file, int line)
{
int wakeup_swapper = 0;
if (__predict_false(panicstr != NULL))
goto out;
wakeup_swapper = wakeupshlk(lk, file, line);
out:
lockmgr_exit(flags, ilk, wakeup_swapper);
return (0);
}
static __noinline int
lockmgr_xunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk,
const char *file, int line)
{
uintptr_t tid, v;
int wakeup_swapper = 0;
u_int realexslp;
int queue;
if (__predict_false(panicstr != NULL))
goto out;
tid = (uintptr_t)curthread;
/*
* As first option, treact the lock as if it has not
* any waiter.
* Fix-up the tid var if the lock has been disowned.
*/
if (LK_HOLDER(x) == LK_KERNPROC)
tid = LK_KERNPROC;
else {
WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
TD_LOCKS_DEC(curthread);
}
LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file, line);
/*
* The lock is held in exclusive mode.
* If the lock is recursed also, then unrecurse it.
*/
if (lockmgr_xlocked_v(x) && lockmgr_recursed(lk)) {
LOCK_LOG2(lk, "%s: %p unrecursing", __func__, lk);
lk->lk_recurse--;
goto out;
}
if (tid != LK_KERNPROC)
LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk,
LOCKSTAT_WRITER);
if (x == tid && atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED))
goto out;
sleepq_lock(&lk->lock_object);
x = lk->lk_lock;
v = LK_UNLOCKED;
/*
* If the lock has exclusive waiters, give them
* preference in order to avoid deadlock with
* shared runners up.
* If interruptible sleeps left the exclusive queue
* empty avoid a starvation for the threads sleeping
* on the shared queue by giving them precedence
* and cleaning up the exclusive waiters bit anyway.
* Please note that lk_exslpfail count may be lying
* about the real number of waiters with the
* LK_SLEEPFAIL flag on because they may be used in
* conjunction with interruptible sleeps so
* lk_exslpfail might be considered an 'upper limit'
* bound, including the edge cases.
*/
MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
realexslp = sleepq_sleepcnt(&lk->lock_object, SQ_EXCLUSIVE_QUEUE);
if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) {
if (lk->lk_exslpfail < realexslp) {
lk->lk_exslpfail = 0;
queue = SQ_EXCLUSIVE_QUEUE;
v |= (x & LK_SHARED_WAITERS);
} else {
lk->lk_exslpfail = 0;
LOCK_LOG2(lk,
"%s: %p has only LK_SLEEPFAIL sleepers",
__func__, lk);
LOCK_LOG2(lk,
"%s: %p waking up threads on the exclusive queue",
__func__, lk);
wakeup_swapper = sleepq_broadcast(&lk->lock_object,
SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE);
queue = SQ_SHARED_QUEUE;
}
} else {
/*
* Exclusive waiters sleeping with LK_SLEEPFAIL
* on and using interruptible sleeps/timeout
* may have left spourious lk_exslpfail counts
* on, so clean it up anyway.
*/
lk->lk_exslpfail = 0;
queue = SQ_SHARED_QUEUE;
}
LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
__func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
"exclusive");
atomic_store_rel_ptr(&lk->lk_lock, v);
wakeup_swapper |= sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue);
sleepq_release(&lk->lock_object);
out:
lockmgr_exit(flags, ilk, wakeup_swapper);
return (0);
}
int
lockmgr_unlock_fast_path(struct lock *lk, u_int flags, struct lock_object *ilk)
{
struct lock_class *class;
uintptr_t x, tid;
const char *file;
int line;
if (__predict_false(panicstr != NULL))
return (0);
file = __FILE__;
line = __LINE__;
_lockmgr_assert(lk, KA_LOCKED, file, line);
x = lk->lk_lock;
if (__predict_true(x & LK_SHARE) != 0) {
if (lockmgr_sunlock_try(lk, &x)) {
lockmgr_note_shared_release(lk, file, line);
} else {
return (lockmgr_sunlock_hard(lk, x, flags, ilk, file, line));
}
} else {
tid = (uintptr_t)curthread;
if (!lockmgr_recursed(lk) &&
atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED)) {
lockmgr_note_exclusive_release(lk, file, line);
} else {
return (lockmgr_xunlock_hard(lk, x, flags, ilk, file, line));
}
}
if (__predict_false(flags & LK_INTERLOCK)) {
class = LOCK_CLASS(ilk);
class->lc_unlock(ilk);
}
return (0);
}
/*
* Lightweight entry points for common operations.
*
* Functionality is similar to sx locks, in that none of the additional lockmgr
* features are supported. To be clear, these are NOT supported:
* 1. shared locking disablement
* 2. returning with an error after sleep
* 3. unlocking the interlock
*
* If in doubt, use lockmgr_*_fast_path.
*/
int
lockmgr_slock(struct lock *lk, u_int flags, const char *file, int line)
{
uintptr_t x;
MPASS((flags & LK_TYPE_MASK) == LK_SHARED);
MPASS((flags & LK_INTERLOCK) == 0);
MPASS((lk->lock_object.lo_flags & LK_NOSHARE) == 0);
if (LK_CAN_WITNESS(flags))
WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
file, line, NULL);
if (__predict_true(lockmgr_slock_try(lk, &x, flags, true))) {
lockmgr_note_shared_acquire(lk, 0, 0, file, line, flags);
return (0);
}
return (lockmgr_slock_hard(lk, flags, NULL, file, line, NULL));
}
int
lockmgr_xlock(struct lock *lk, u_int flags, const char *file, int line)
{
uintptr_t tid;
MPASS((flags & LK_TYPE_MASK) == LK_EXCLUSIVE);
MPASS((flags & LK_INTERLOCK) == 0);
if (LK_CAN_WITNESS(flags))
WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
LOP_EXCLUSIVE, file, line, NULL);
tid = (uintptr_t)curthread;
if (atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) {
lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
flags);
return (0);
}
return (lockmgr_xlock_hard(lk, flags, NULL, file, line, NULL));
}
int
lockmgr_unlock(struct lock *lk)
{
uintptr_t x, tid;
const char *file;
int line;
file = __FILE__;
line = __LINE__;
_lockmgr_assert(lk, KA_LOCKED, file, line);
x = lk->lk_lock;
if (__predict_true(x & LK_SHARE) != 0) {
if (lockmgr_sunlock_try(lk, &x)) {
lockmgr_note_shared_release(lk, file, line);
} else {
return (lockmgr_sunlock_hard(lk, x, LK_RELEASE, NULL, file, line));
}
} else {
tid = (uintptr_t)curthread;
if (!lockmgr_recursed(lk) &&
atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED)) {
lockmgr_note_exclusive_release(lk, file, line);
} else {
return (lockmgr_xunlock_hard(lk, x, LK_RELEASE, NULL, file, line));
}
}
return (0);
}
int
__lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
const char *wmesg, int pri, int timo, const char *file, int line)
{
GIANT_DECLARE;
struct lockmgr_wait lwa;
struct lock_class *class;
const char *iwmesg;
uintptr_t tid, v, x;
u_int op, realexslp;
int error, ipri, itimo, queue, wakeup_swapper;
#ifdef LOCK_PROFILING
uint64_t waittime = 0;
int contested = 0;
#endif
if (panicstr != NULL)
return (0);
error = 0;
tid = (uintptr_t)curthread;
op = (flags & LK_TYPE_MASK);
iwmesg = (wmesg == LK_WMESG_DEFAULT) ? lk->lock_object.lo_name : wmesg;
ipri = (pri == LK_PRIO_DEFAULT) ? lk->lk_pri : pri;
itimo = (timo == LK_TIMO_DEFAULT) ? lk->lk_timo : timo;
lwa.iwmesg = iwmesg;
lwa.ipri = ipri;
lwa.itimo = itimo;
MPASS((flags & ~LK_TOTAL_MASK) == 0);
KASSERT((op & (op - 1)) == 0,
("%s: Invalid requested operation @ %s:%d", __func__, file, line));
KASSERT((flags & (LK_NOWAIT | LK_SLEEPFAIL)) == 0 ||
(op != LK_DOWNGRADE && op != LK_RELEASE),
("%s: Invalid flags in regard of the operation desired @ %s:%d",
__func__, file, line));
KASSERT((flags & LK_INTERLOCK) == 0 || ilk != NULL,
("%s: LK_INTERLOCK passed without valid interlock @ %s:%d",
__func__, file, line));
KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
("%s: idle thread %p on lockmgr %s @ %s:%d", __func__, curthread,
lk->lock_object.lo_name, file, line));
class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
if (lk->lock_object.lo_flags & LK_NOSHARE) {
switch (op) {
case LK_SHARED:
op = LK_EXCLUSIVE;
break;
case LK_UPGRADE:
case LK_TRYUPGRADE:
case LK_DOWNGRADE:
_lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED,
file, line);
if (flags & LK_INTERLOCK)
class->lc_unlock(ilk);
return (0);
}
}
wakeup_swapper = 0;
switch (op) {
case LK_SHARED:
return (lockmgr_slock_hard(lk, flags, ilk, file, line, &lwa));
break;
case LK_UPGRADE:
case LK_TRYUPGRADE:
return (lockmgr_upgrade(lk, flags, ilk, file, line, &lwa));
break;
case LK_EXCLUSIVE:
return (lockmgr_xlock_hard(lk, flags, ilk, file, line, &lwa));
break;
case LK_DOWNGRADE:
_lockmgr_assert(lk, KA_XLOCKED, file, line);
WITNESS_DOWNGRADE(&lk->lock_object, 0, file, line);
/*
* Panic if the lock is recursed.
*/
if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) {
if (flags & LK_INTERLOCK)
class->lc_unlock(ilk);
panic("%s: downgrade a recursed lockmgr %s @ %s:%d\n",
__func__, iwmesg, file, line);
}
TD_SLOCKS_INC(curthread);
/*
* In order to preserve waiters flags, just spin.
*/
for (;;) {
x = lk->lk_lock;
MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
x &= LK_ALL_WAITERS;
if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
LK_SHARERS_LOCK(1) | x))
break;
cpu_spinwait();
}
LOCK_LOG_LOCK("XDOWNGRADE", &lk->lock_object, 0, 0, file, line);
LOCKSTAT_RECORD0(lockmgr__downgrade, lk);
break;
case LK_RELEASE:
_lockmgr_assert(lk, KA_LOCKED, file, line);
x = lk->lk_lock;
if (__predict_true(x & LK_SHARE) != 0) {
return (lockmgr_sunlock_hard(lk, x, flags, ilk, file, line));
} else {
return (lockmgr_xunlock_hard(lk, x, flags, ilk, file, line));
}
break;
case LK_DRAIN:
if (LK_CAN_WITNESS(flags))
WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ?
ilk : NULL);
/*
* Trying to drain a lock we already own will result in a
* deadlock.
*/
if (lockmgr_xlocked(lk)) {
if (flags & LK_INTERLOCK)
class->lc_unlock(ilk);
panic("%s: draining %s with the lock held @ %s:%d\n",
__func__, iwmesg, file, line);
}
for (;;) {
if (lk->lk_lock == LK_UNLOCKED &&
atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid))
break;
#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);
#endif
lock_profile_obtain_lock_failed(&lk->lock_object,
&contested, &waittime);
/*
* If the lock is expected to not sleep just give up
* and return.
*/
if (LK_TRYOP(flags)) {
LOCK_LOG2(lk, "%s: %p fails the try operation",
__func__, lk);
error = EBUSY;
break;
}
/*
* Acquire the sleepqueue chain lock because we
* probabilly will need to manipulate waiters flags.
*/
sleepq_lock(&lk->lock_object);
x = lk->lk_lock;
/*
* if the lock has been released while we spun on
* the sleepqueue chain lock just try again.
*/
if (x == LK_UNLOCKED) {
sleepq_release(&lk->lock_object);
continue;
}
v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
if ((x & ~v) == LK_UNLOCKED) {
v = (x & ~LK_EXCLUSIVE_SPINNERS);
/*
* If interruptible sleeps left the exclusive
* queue empty avoid a starvation for the
* threads sleeping on the shared queue by
* giving them precedence and cleaning up the
* exclusive waiters bit anyway.
* Please note that lk_exslpfail count may be
* lying about the real number of waiters with
* the LK_SLEEPFAIL flag on because they may
* be used in conjunction with interruptible
* sleeps so lk_exslpfail might be considered
* an 'upper limit' bound, including the edge
* cases.
*/
if (v & LK_EXCLUSIVE_WAITERS) {
queue = SQ_EXCLUSIVE_QUEUE;
v &= ~LK_EXCLUSIVE_WAITERS;
} else {
/*
* Exclusive waiters sleeping with
* LK_SLEEPFAIL on and using
* interruptible sleeps/timeout may
* have left spourious lk_exslpfail
* counts on, so clean it up anyway.
*/
MPASS(v & LK_SHARED_WAITERS);
lk->lk_exslpfail = 0;
queue = SQ_SHARED_QUEUE;
v &= ~LK_SHARED_WAITERS;
}
if (queue == SQ_EXCLUSIVE_QUEUE) {
realexslp =
sleepq_sleepcnt(&lk->lock_object,
SQ_EXCLUSIVE_QUEUE);
if (lk->lk_exslpfail >= realexslp) {
lk->lk_exslpfail = 0;
queue = SQ_SHARED_QUEUE;
v &= ~LK_SHARED_WAITERS;
if (realexslp != 0) {
LOCK_LOG2(lk,
"%s: %p has only LK_SLEEPFAIL sleepers",
__func__, lk);
LOCK_LOG2(lk,
"%s: %p waking up threads on the exclusive queue",
__func__, lk);
wakeup_swapper =
sleepq_broadcast(
&lk->lock_object,
SLEEPQ_LK, 0,
SQ_EXCLUSIVE_QUEUE);
}
} else
lk->lk_exslpfail = 0;
}
if (!atomic_cmpset_ptr(&lk->lk_lock, x, v)) {
sleepq_release(&lk->lock_object);
continue;
}
LOCK_LOG3(lk,
"%s: %p waking up all threads on the %s queue",
__func__, lk, queue == SQ_SHARED_QUEUE ?
"shared" : "exclusive");
wakeup_swapper |= sleepq_broadcast(
&lk->lock_object, SLEEPQ_LK, 0, queue);
/*
* If shared waiters have been woken up we need
* to wait for one of them to acquire the lock
* before to set the exclusive waiters in
* order to avoid a deadlock.
*/
if (queue == SQ_SHARED_QUEUE) {
for (v = lk->lk_lock;
(v & LK_SHARE) && !LK_SHARERS(v);
v = lk->lk_lock)
cpu_spinwait();
}
}
/*
* Try to set the LK_EXCLUSIVE_WAITERS flag. If we
* fail, loop back and retry.
*/
if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
if (!atomic_cmpset_ptr(&lk->lk_lock, x,
x | LK_EXCLUSIVE_WAITERS)) {
sleepq_release(&lk->lock_object);
continue;
}
LOCK_LOG2(lk, "%s: %p set drain waiters flag",
__func__, lk);
}
/*
* As far as we have been unable to acquire the
* exclusive lock and the exclusive waiters flag
* is set, we will sleep.
*/
if (flags & LK_INTERLOCK) {
class->lc_unlock(ilk);
flags &= ~LK_INTERLOCK;
}
GIANT_SAVE();
sleepq_add(&lk->lock_object, NULL, iwmesg, SLEEPQ_LK,
SQ_EXCLUSIVE_QUEUE);
sleepq_wait(&lk->lock_object, ipri & PRIMASK);
GIANT_RESTORE();
LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
__func__, lk);
}
if (error == 0) {
lock_profile_obtain_lock_success(&lk->lock_object,
contested, waittime, file, line);
LOCK_LOG_LOCK("DRAIN", &lk->lock_object, 0,
lk->lk_recurse, file, line);
WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE |
LK_TRYWIT(flags), file, line);
TD_LOCKS_INC(curthread);
STACK_SAVE(lk);
}
break;
default:
if (flags & LK_INTERLOCK)
class->lc_unlock(ilk);
panic("%s: unknown lockmgr request 0x%x\n", __func__, op);
}
if (flags & LK_INTERLOCK)
class->lc_unlock(ilk);
if (wakeup_swapper)
kick_proc0();
return (error);
}
void
_lockmgr_disown(struct lock *lk, const char *file, int line)
{
uintptr_t tid, x;
if (SCHEDULER_STOPPED())
return;
tid = (uintptr_t)curthread;
_lockmgr_assert(lk, KA_XLOCKED, file, line);
/*
* Panic if the lock is recursed.
*/
if (lockmgr_xlocked(lk) && lockmgr_recursed(lk))
panic("%s: disown a recursed lockmgr @ %s:%d\n",
__func__, file, line);
/*
* If the owner is already LK_KERNPROC just skip the whole operation.
*/
if (LK_HOLDER(lk->lk_lock) != tid)
return;
lock_profile_release_lock(&lk->lock_object);
LOCKSTAT_RECORD1(lockmgr__disown, lk, LOCKSTAT_WRITER);
LOCK_LOG_LOCK("XDISOWN", &lk->lock_object, 0, 0, file, line);
WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
TD_LOCKS_DEC(curthread);
STACK_SAVE(lk);
/*
* In order to preserve waiters flags, just spin.
*/
for (;;) {
x = lk->lk_lock;
MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
x &= LK_ALL_WAITERS;
if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
LK_KERNPROC | x))
return;
cpu_spinwait();
}
}
void
lockmgr_printinfo(const struct lock *lk)
{
struct thread *td;
uintptr_t x;
if (lk->lk_lock == LK_UNLOCKED)
printf("lock type %s: UNLOCKED\n", lk->lock_object.lo_name);
else if (lk->lk_lock & LK_SHARE)
printf("lock type %s: SHARED (count %ju)\n",
lk->lock_object.lo_name,
(uintmax_t)LK_SHARERS(lk->lk_lock));
else {
td = lockmgr_xholder(lk);
if (td == (struct thread *)LK_KERNPROC)
printf("lock type %s: EXCL by KERNPROC\n",
lk->lock_object.lo_name);
else
printf("lock type %s: EXCL by thread %p "
"(pid %d, %s, tid %d)\n", lk->lock_object.lo_name,
td, td->td_proc->p_pid, td->td_proc->p_comm,
td->td_tid);
}
x = lk->lk_lock;
if (x & LK_EXCLUSIVE_WAITERS)
printf(" with exclusive waiters pending\n");
if (x & LK_SHARED_WAITERS)
printf(" with shared waiters pending\n");
if (x & LK_EXCLUSIVE_SPINNERS)
printf(" with exclusive spinners pending\n");
STACK_PRINT(lk);
}
int
lockstatus(const struct lock *lk)
{
uintptr_t v, x;
int ret;
ret = LK_SHARED;
x = lk->lk_lock;
v = LK_HOLDER(x);
if ((x & LK_SHARE) == 0) {
if (v == (uintptr_t)curthread || v == LK_KERNPROC)
ret = LK_EXCLUSIVE;
else
ret = LK_EXCLOTHER;
} else if (x == LK_UNLOCKED)
ret = 0;
return (ret);
}
#ifdef INVARIANT_SUPPORT
FEATURE(invariant_support,
"Support for modules compiled with INVARIANTS option");
#ifndef INVARIANTS
#undef _lockmgr_assert
#endif
void
_lockmgr_assert(const struct lock *lk, int what, const char *file, int line)
{
int slocked = 0;
if (panicstr != NULL)
return;
switch (what) {
case KA_SLOCKED:
case KA_SLOCKED | KA_NOTRECURSED:
case KA_SLOCKED | KA_RECURSED:
slocked = 1;
case KA_LOCKED:
case KA_LOCKED | KA_NOTRECURSED:
case KA_LOCKED | KA_RECURSED:
#ifdef WITNESS
/*
* We cannot trust WITNESS if the lock is held in exclusive
* mode and a call to lockmgr_disown() happened.
* Workaround this skipping the check if the lock is held in
* exclusive mode even for the KA_LOCKED case.
*/
if (slocked || (lk->lk_lock & LK_SHARE)) {
witness_assert(&lk->lock_object, what, file, line);
break;
}
#endif
if (lk->lk_lock == LK_UNLOCKED ||
((lk->lk_lock & LK_SHARE) == 0 && (slocked ||
(!lockmgr_xlocked(lk) && !lockmgr_disowned(lk)))))
panic("Lock %s not %slocked @ %s:%d\n",
lk->lock_object.lo_name, slocked ? "share" : "",
file, line);
if ((lk->lk_lock & LK_SHARE) == 0) {
if (lockmgr_recursed(lk)) {
if (what & KA_NOTRECURSED)
panic("Lock %s recursed @ %s:%d\n",
lk->lock_object.lo_name, file,
line);
} else if (what & KA_RECURSED)
panic("Lock %s not recursed @ %s:%d\n",
lk->lock_object.lo_name, file, line);
}
break;
case KA_XLOCKED:
case KA_XLOCKED | KA_NOTRECURSED:
case KA_XLOCKED | KA_RECURSED:
if (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk))
panic("Lock %s not exclusively locked @ %s:%d\n",
lk->lock_object.lo_name, file, line);
if (lockmgr_recursed(lk)) {
if (what & KA_NOTRECURSED)
panic("Lock %s recursed @ %s:%d\n",
lk->lock_object.lo_name, file, line);
} else if (what & KA_RECURSED)
panic("Lock %s not recursed @ %s:%d\n",
lk->lock_object.lo_name, file, line);
break;
case KA_UNLOCKED:
if (lockmgr_xlocked(lk) || lockmgr_disowned(lk))
panic("Lock %s exclusively locked @ %s:%d\n",
lk->lock_object.lo_name, file, line);
break;
default:
panic("Unknown lockmgr assertion: %d @ %s:%d\n", what, file,
line);
}
}
#endif
#ifdef DDB
int
lockmgr_chain(struct thread *td, struct thread **ownerp)
{
- struct lock *lk;
+ const struct lock *lk;
lk = td->td_wchan;
if (LOCK_CLASS(&lk->lock_object) != &lock_class_lockmgr)
return (0);
db_printf("blocked on lockmgr %s", lk->lock_object.lo_name);
if (lk->lk_lock & LK_SHARE)
db_printf("SHARED (count %ju)\n",
(uintmax_t)LK_SHARERS(lk->lk_lock));
else
db_printf("EXCL\n");
*ownerp = lockmgr_xholder(lk);
return (1);
}
static void
db_show_lockmgr(const struct lock_object *lock)
{
struct thread *td;
const struct lock *lk;
lk = (const struct lock *)lock;
db_printf(" state: ");
if (lk->lk_lock == LK_UNLOCKED)
db_printf("UNLOCKED\n");
else if (lk->lk_lock & LK_SHARE)
db_printf("SLOCK: %ju\n", (uintmax_t)LK_SHARERS(lk->lk_lock));
else {
td = lockmgr_xholder(lk);
if (td == (struct thread *)LK_KERNPROC)
db_printf("XLOCK: LK_KERNPROC\n");
else
db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
td->td_tid, td->td_proc->p_pid,
td->td_proc->p_comm);
if (lockmgr_recursed(lk))
db_printf(" recursed: %d\n", lk->lk_recurse);
}
db_printf(" waiters: ");
switch (lk->lk_lock & LK_ALL_WAITERS) {
case LK_SHARED_WAITERS:
db_printf("shared\n");
break;
case LK_EXCLUSIVE_WAITERS:
db_printf("exclusive\n");
break;
case LK_ALL_WAITERS:
db_printf("shared and exclusive\n");
break;
default:
db_printf("none\n");
}
db_printf(" spinners: ");
if (lk->lk_lock & LK_EXCLUSIVE_SPINNERS)
db_printf("exclusive\n");
else
db_printf("none\n");
}
#endif
Index: head/sys/kern/kern_proc.c
===================================================================
--- head/sys/kern/kern_proc.c (revision 356056)
+++ head/sys/kern/kern_proc.c (revision 356057)
@@ -1,3221 +1,3221 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1982, 1986, 1989, 1991, 1993
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)kern_proc.c 8.7 (Berkeley) 2/14/95
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_ddb.h"
#include "opt_ktrace.h"
#include "opt_kstack_pages.h"
#include "opt_stack.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bitstring.h>
#include <sys/elf.h>
#include <sys/eventhandler.h>
#include <sys/exec.h>
#include <sys/jail.h>
#include <sys/kernel.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/loginclass.h>
#include <sys/malloc.h>
#include <sys/mman.h>
#include <sys/mount.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/ptrace.h>
#include <sys/refcount.h>
#include <sys/resourcevar.h>
#include <sys/rwlock.h>
#include <sys/sbuf.h>
#include <sys/sysent.h>
#include <sys/sched.h>
#include <sys/smp.h>
#include <sys/stack.h>
#include <sys/stat.h>
#include <sys/sysctl.h>
#include <sys/filedesc.h>
#include <sys/tty.h>
#include <sys/signalvar.h>
#include <sys/sdt.h>
#include <sys/sx.h>
#include <sys/user.h>
#include <sys/vnode.h>
#include <sys/wait.h>
#ifdef DDB
#include <ddb/ddb.h>
#endif
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/vm_extern.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/uma.h>
#ifdef COMPAT_FREEBSD32
#include <compat/freebsd32/freebsd32.h>
#include <compat/freebsd32/freebsd32_util.h>
#endif
SDT_PROVIDER_DEFINE(proc);
MALLOC_DEFINE(M_PGRP, "pgrp", "process group header");
MALLOC_DEFINE(M_SESSION, "session", "session header");
static MALLOC_DEFINE(M_PROC, "proc", "Proc structures");
MALLOC_DEFINE(M_SUBPROC, "subproc", "Proc sub-structures");
static void doenterpgrp(struct proc *, struct pgrp *);
static void orphanpg(struct pgrp *pg);
static void fill_kinfo_aggregate(struct proc *p, struct kinfo_proc *kp);
static void fill_kinfo_proc_only(struct proc *p, struct kinfo_proc *kp);
static void fill_kinfo_thread(struct thread *td, struct kinfo_proc *kp,
int preferthread);
static void pgadjustjobc(struct pgrp *pgrp, int entering);
static void pgdelete(struct pgrp *);
static int proc_ctor(void *mem, int size, void *arg, int flags);
static void proc_dtor(void *mem, int size, void *arg);
static int proc_init(void *mem, int size, int flags);
static void proc_fini(void *mem, int size);
static void pargs_free(struct pargs *pa);
/*
* Other process lists
*/
struct pidhashhead *pidhashtbl;
struct sx *pidhashtbl_lock;
u_long pidhash;
u_long pidhashlock;
struct pgrphashhead *pgrphashtbl;
u_long pgrphash;
struct proclist allproc;
struct sx __exclusive_cache_line allproc_lock;
struct sx __exclusive_cache_line proctree_lock;
struct mtx __exclusive_cache_line ppeers_lock;
struct mtx __exclusive_cache_line procid_lock;
uma_zone_t proc_zone;
/*
* The offset of various fields in struct proc and struct thread.
* These are used by kernel debuggers to enumerate kernel threads and
* processes.
*/
const int proc_off_p_pid = offsetof(struct proc, p_pid);
const int proc_off_p_comm = offsetof(struct proc, p_comm);
const int proc_off_p_list = offsetof(struct proc, p_list);
const int proc_off_p_threads = offsetof(struct proc, p_threads);
const int thread_off_td_tid = offsetof(struct thread, td_tid);
const int thread_off_td_name = offsetof(struct thread, td_name);
const int thread_off_td_oncpu = offsetof(struct thread, td_oncpu);
const int thread_off_td_pcb = offsetof(struct thread, td_pcb);
const int thread_off_td_plist = offsetof(struct thread, td_plist);
EVENTHANDLER_LIST_DEFINE(process_ctor);
EVENTHANDLER_LIST_DEFINE(process_dtor);
EVENTHANDLER_LIST_DEFINE(process_init);
EVENTHANDLER_LIST_DEFINE(process_fini);
EVENTHANDLER_LIST_DEFINE(process_exit);
EVENTHANDLER_LIST_DEFINE(process_fork);
EVENTHANDLER_LIST_DEFINE(process_exec);
int kstack_pages = KSTACK_PAGES;
SYSCTL_INT(_kern, OID_AUTO, kstack_pages, CTLFLAG_RD, &kstack_pages, 0,
"Kernel stack size in pages");
static int vmmap_skip_res_cnt = 0;
SYSCTL_INT(_kern, OID_AUTO, proc_vmmap_skip_resident_count, CTLFLAG_RW,
&vmmap_skip_res_cnt, 0,
"Skip calculation of the pages resident count in kern.proc.vmmap");
CTASSERT(sizeof(struct kinfo_proc) == KINFO_PROC_SIZE);
#ifdef COMPAT_FREEBSD32
CTASSERT(sizeof(struct kinfo_proc32) == KINFO_PROC32_SIZE);
#endif
/*
* Initialize global process hashing structures.
*/
void
procinit(void)
{
u_long i;
sx_init(&allproc_lock, "allproc");
sx_init(&proctree_lock, "proctree");
mtx_init(&ppeers_lock, "p_peers", NULL, MTX_DEF);
mtx_init(&procid_lock, "procid", NULL, MTX_DEF);
LIST_INIT(&allproc);
pidhashtbl = hashinit(maxproc / 4, M_PROC, &pidhash);
pidhashlock = (pidhash + 1) / 64;
if (pidhashlock > 0)
pidhashlock--;
pidhashtbl_lock = malloc(sizeof(*pidhashtbl_lock) * (pidhashlock + 1),
M_PROC, M_WAITOK | M_ZERO);
for (i = 0; i < pidhashlock + 1; i++)
sx_init_flags(&pidhashtbl_lock[i], "pidhash", SX_DUPOK);
pgrphashtbl = hashinit(maxproc / 4, M_PROC, &pgrphash);
proc_zone = uma_zcreate("PROC", sched_sizeof_proc(),
proc_ctor, proc_dtor, proc_init, proc_fini,
UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
uihashinit();
}
/*
* Prepare a proc for use.
*/
static int
proc_ctor(void *mem, int size, void *arg, int flags)
{
struct proc *p;
struct thread *td;
p = (struct proc *)mem;
EVENTHANDLER_DIRECT_INVOKE(process_ctor, p);
td = FIRST_THREAD_IN_PROC(p);
if (td != NULL) {
/* Make sure all thread constructors are executed */
EVENTHANDLER_DIRECT_INVOKE(thread_ctor, td);
}
return (0);
}
/*
* Reclaim a proc after use.
*/
static void
proc_dtor(void *mem, int size, void *arg)
{
struct proc *p;
struct thread *td;
/* INVARIANTS checks go here */
p = (struct proc *)mem;
td = FIRST_THREAD_IN_PROC(p);
if (td != NULL) {
#ifdef INVARIANTS
KASSERT((p->p_numthreads == 1),
("bad number of threads in exiting process"));
KASSERT(STAILQ_EMPTY(&p->p_ktr), ("proc_dtor: non-empty p_ktr"));
#endif
/* Free all OSD associated to this thread. */
osd_thread_exit(td);
td_softdep_cleanup(td);
MPASS(td->td_su == NULL);
/* Make sure all thread destructors are executed */
EVENTHANDLER_DIRECT_INVOKE(thread_dtor, td);
}
EVENTHANDLER_DIRECT_INVOKE(process_dtor, p);
if (p->p_ksi != NULL)
KASSERT(! KSI_ONQ(p->p_ksi), ("SIGCHLD queue"));
}
/*
* Initialize type-stable parts of a proc (when newly created).
*/
static int
proc_init(void *mem, int size, int flags)
{
struct proc *p;
p = (struct proc *)mem;
mtx_init(&p->p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK | MTX_NEW);
mtx_init(&p->p_slock, "process slock", NULL, MTX_SPIN | MTX_NEW);
mtx_init(&p->p_statmtx, "pstatl", NULL, MTX_SPIN | MTX_NEW);
mtx_init(&p->p_itimmtx, "pitiml", NULL, MTX_SPIN | MTX_NEW);
mtx_init(&p->p_profmtx, "pprofl", NULL, MTX_SPIN | MTX_NEW);
cv_init(&p->p_pwait, "ppwait");
TAILQ_INIT(&p->p_threads); /* all threads in proc */
EVENTHANDLER_DIRECT_INVOKE(process_init, p);
p->p_stats = pstats_alloc();
p->p_pgrp = NULL;
return (0);
}
/*
* UMA should ensure that this function is never called.
* Freeing a proc structure would violate type stability.
*/
static void
proc_fini(void *mem, int size)
{
#ifdef notnow
struct proc *p;
p = (struct proc *)mem;
EVENTHANDLER_DIRECT_INVOKE(process_fini, p);
pstats_free(p->p_stats);
thread_free(FIRST_THREAD_IN_PROC(p));
mtx_destroy(&p->p_mtx);
if (p->p_ksi != NULL)
ksiginfo_free(p->p_ksi);
#else
panic("proc reclaimed");
#endif
}
/*
* PID space management.
*
* These bitmaps are used by fork_findpid.
*/
bitstr_t bit_decl(proc_id_pidmap, PID_MAX);
bitstr_t bit_decl(proc_id_grpidmap, PID_MAX);
bitstr_t bit_decl(proc_id_sessidmap, PID_MAX);
bitstr_t bit_decl(proc_id_reapmap, PID_MAX);
static bitstr_t *proc_id_array[] = {
proc_id_pidmap,
proc_id_grpidmap,
proc_id_sessidmap,
proc_id_reapmap,
};
void
proc_id_set(int type, pid_t id)
{
KASSERT(type >= 0 && type < nitems(proc_id_array),
("invalid type %d\n", type));
mtx_lock(&procid_lock);
KASSERT(bit_test(proc_id_array[type], id) == 0,
("bit %d already set in %d\n", id, type));
bit_set(proc_id_array[type], id);
mtx_unlock(&procid_lock);
}
void
proc_id_set_cond(int type, pid_t id)
{
KASSERT(type >= 0 && type < nitems(proc_id_array),
("invalid type %d\n", type));
if (bit_test(proc_id_array[type], id))
return;
mtx_lock(&procid_lock);
bit_set(proc_id_array[type], id);
mtx_unlock(&procid_lock);
}
void
proc_id_clear(int type, pid_t id)
{
KASSERT(type >= 0 && type < nitems(proc_id_array),
("invalid type %d\n", type));
mtx_lock(&procid_lock);
KASSERT(bit_test(proc_id_array[type], id) != 0,
("bit %d not set in %d\n", id, type));
bit_clear(proc_id_array[type], id);
mtx_unlock(&procid_lock);
}
/*
* Is p an inferior of the current process?
*/
int
inferior(struct proc *p)
{
sx_assert(&proctree_lock, SX_LOCKED);
PROC_LOCK_ASSERT(p, MA_OWNED);
for (; p != curproc; p = proc_realparent(p)) {
if (p->p_pid == 0)
return (0);
}
return (1);
}
/*
* Shared lock all the pid hash lists.
*/
void
pidhash_slockall(void)
{
u_long i;
for (i = 0; i < pidhashlock + 1; i++)
sx_slock(&pidhashtbl_lock[i]);
}
/*
* Shared unlock all the pid hash lists.
*/
void
pidhash_sunlockall(void)
{
u_long i;
for (i = 0; i < pidhashlock + 1; i++)
sx_sunlock(&pidhashtbl_lock[i]);
}
/*
* Similar to pfind_any(), this function finds zombies.
*/
struct proc *
pfind_any_locked(pid_t pid)
{
struct proc *p;
sx_assert(PIDHASHLOCK(pid), SX_LOCKED);
LIST_FOREACH(p, PIDHASH(pid), p_hash) {
if (p->p_pid == pid) {
PROC_LOCK(p);
if (p->p_state == PRS_NEW) {
PROC_UNLOCK(p);
p = NULL;
}
break;
}
}
return (p);
}
/*
* Locate a process by number.
*
* By not returning processes in the PRS_NEW state, we allow callers to avoid
* testing for that condition to avoid dereferencing p_ucred, et al.
*/
static __always_inline struct proc *
_pfind(pid_t pid, bool zombie)
{
struct proc *p;
p = curproc;
if (p->p_pid == pid) {
PROC_LOCK(p);
return (p);
}
sx_slock(PIDHASHLOCK(pid));
LIST_FOREACH(p, PIDHASH(pid), p_hash) {
if (p->p_pid == pid) {
PROC_LOCK(p);
if (p->p_state == PRS_NEW ||
(!zombie && p->p_state == PRS_ZOMBIE)) {
PROC_UNLOCK(p);
p = NULL;
}
break;
}
}
sx_sunlock(PIDHASHLOCK(pid));
return (p);
}
struct proc *
pfind(pid_t pid)
{
return (_pfind(pid, false));
}
/*
* Same as pfind but allow zombies.
*/
struct proc *
pfind_any(pid_t pid)
{
return (_pfind(pid, true));
}
static struct proc *
pfind_tid(pid_t tid)
{
struct proc *p;
struct thread *td;
sx_slock(&allproc_lock);
FOREACH_PROC_IN_SYSTEM(p) {
PROC_LOCK(p);
if (p->p_state == PRS_NEW) {
PROC_UNLOCK(p);
continue;
}
FOREACH_THREAD_IN_PROC(p, td) {
if (td->td_tid == tid)
goto found;
}
PROC_UNLOCK(p);
}
found:
sx_sunlock(&allproc_lock);
return (p);
}
/*
* Locate a process group by number.
* The caller must hold proctree_lock.
*/
struct pgrp *
pgfind(pid_t pgid)
{
struct pgrp *pgrp;
sx_assert(&proctree_lock, SX_LOCKED);
LIST_FOREACH(pgrp, PGRPHASH(pgid), pg_hash) {
if (pgrp->pg_id == pgid) {
PGRP_LOCK(pgrp);
return (pgrp);
}
}
return (NULL);
}
/*
* Locate process and do additional manipulations, depending on flags.
*/
int
pget(pid_t pid, int flags, struct proc **pp)
{
struct proc *p;
int error;
p = curproc;
if (p->p_pid == pid) {
PROC_LOCK(p);
} else {
p = NULL;
if (pid <= PID_MAX) {
if ((flags & PGET_NOTWEXIT) == 0)
p = pfind_any(pid);
else
p = pfind(pid);
} else if ((flags & PGET_NOTID) == 0) {
p = pfind_tid(pid);
}
if (p == NULL)
return (ESRCH);
if ((flags & PGET_CANSEE) != 0) {
error = p_cansee(curthread, p);
if (error != 0)
goto errout;
}
}
if ((flags & PGET_CANDEBUG) != 0) {
error = p_candebug(curthread, p);
if (error != 0)
goto errout;
}
if ((flags & PGET_ISCURRENT) != 0 && curproc != p) {
error = EPERM;
goto errout;
}
if ((flags & PGET_NOTWEXIT) != 0 && (p->p_flag & P_WEXIT) != 0) {
error = ESRCH;
goto errout;
}
if ((flags & PGET_NOTINEXEC) != 0 && (p->p_flag & P_INEXEC) != 0) {
/*
* XXXRW: Not clear ESRCH is the right error during proc
* execve().
*/
error = ESRCH;
goto errout;
}
if ((flags & PGET_HOLD) != 0) {
_PHOLD(p);
PROC_UNLOCK(p);
}
*pp = p;
return (0);
errout:
PROC_UNLOCK(p);
return (error);
}
/*
* Create a new process group.
* pgid must be equal to the pid of p.
* Begin a new session if required.
*/
int
enterpgrp(struct proc *p, pid_t pgid, struct pgrp *pgrp, struct session *sess)
{
sx_assert(&proctree_lock, SX_XLOCKED);
KASSERT(pgrp != NULL, ("enterpgrp: pgrp == NULL"));
KASSERT(p->p_pid == pgid,
("enterpgrp: new pgrp and pid != pgid"));
KASSERT(pgfind(pgid) == NULL,
("enterpgrp: pgrp with pgid exists"));
KASSERT(!SESS_LEADER(p),
("enterpgrp: session leader attempted setpgrp"));
mtx_init(&pgrp->pg_mtx, "process group", NULL, MTX_DEF | MTX_DUPOK);
if (sess != NULL) {
/*
* new session
*/
mtx_init(&sess->s_mtx, "session", NULL, MTX_DEF);
PROC_LOCK(p);
p->p_flag &= ~P_CONTROLT;
PROC_UNLOCK(p);
PGRP_LOCK(pgrp);
sess->s_leader = p;
sess->s_sid = p->p_pid;
proc_id_set(PROC_ID_SESSION, p->p_pid);
refcount_init(&sess->s_count, 1);
sess->s_ttyvp = NULL;
sess->s_ttydp = NULL;
sess->s_ttyp = NULL;
bcopy(p->p_session->s_login, sess->s_login,
sizeof(sess->s_login));
pgrp->pg_session = sess;
KASSERT(p == curproc,
("enterpgrp: mksession and p != curproc"));
} else {
pgrp->pg_session = p->p_session;
sess_hold(pgrp->pg_session);
PGRP_LOCK(pgrp);
}
pgrp->pg_id = pgid;
proc_id_set(PROC_ID_GROUP, p->p_pid);
LIST_INIT(&pgrp->pg_members);
/*
* As we have an exclusive lock of proctree_lock,
* this should not deadlock.
*/
LIST_INSERT_HEAD(PGRPHASH(pgid), pgrp, pg_hash);
pgrp->pg_jobc = 0;
SLIST_INIT(&pgrp->pg_sigiolst);
PGRP_UNLOCK(pgrp);
doenterpgrp(p, pgrp);
return (0);
}
/*
* Move p to an existing process group
*/
int
enterthispgrp(struct proc *p, struct pgrp *pgrp)
{
sx_assert(&proctree_lock, SX_XLOCKED);
PROC_LOCK_ASSERT(p, MA_NOTOWNED);
PGRP_LOCK_ASSERT(pgrp, MA_NOTOWNED);
PGRP_LOCK_ASSERT(p->p_pgrp, MA_NOTOWNED);
SESS_LOCK_ASSERT(p->p_session, MA_NOTOWNED);
KASSERT(pgrp->pg_session == p->p_session,
("%s: pgrp's session %p, p->p_session %p.\n",
__func__,
pgrp->pg_session,
p->p_session));
KASSERT(pgrp != p->p_pgrp,
("%s: p belongs to pgrp.", __func__));
doenterpgrp(p, pgrp);
return (0);
}
/*
* Move p to a process group
*/
static void
doenterpgrp(struct proc *p, struct pgrp *pgrp)
{
struct pgrp *savepgrp;
sx_assert(&proctree_lock, SX_XLOCKED);
PROC_LOCK_ASSERT(p, MA_NOTOWNED);
PGRP_LOCK_ASSERT(pgrp, MA_NOTOWNED);
PGRP_LOCK_ASSERT(p->p_pgrp, MA_NOTOWNED);
SESS_LOCK_ASSERT(p->p_session, MA_NOTOWNED);
savepgrp = p->p_pgrp;
/*
* Adjust eligibility of affected pgrps to participate in job control.
* Increment eligibility counts before decrementing, otherwise we
* could reach 0 spuriously during the first call.
*/
fixjobc(p, pgrp, 1);
fixjobc(p, p->p_pgrp, 0);
PGRP_LOCK(pgrp);
PGRP_LOCK(savepgrp);
PROC_LOCK(p);
LIST_REMOVE(p, p_pglist);
p->p_pgrp = pgrp;
PROC_UNLOCK(p);
LIST_INSERT_HEAD(&pgrp->pg_members, p, p_pglist);
PGRP_UNLOCK(savepgrp);
PGRP_UNLOCK(pgrp);
if (LIST_EMPTY(&savepgrp->pg_members))
pgdelete(savepgrp);
}
/*
* remove process from process group
*/
int
leavepgrp(struct proc *p)
{
struct pgrp *savepgrp;
sx_assert(&proctree_lock, SX_XLOCKED);
savepgrp = p->p_pgrp;
PGRP_LOCK(savepgrp);
PROC_LOCK(p);
LIST_REMOVE(p, p_pglist);
p->p_pgrp = NULL;
PROC_UNLOCK(p);
PGRP_UNLOCK(savepgrp);
if (LIST_EMPTY(&savepgrp->pg_members))
pgdelete(savepgrp);
return (0);
}
/*
* delete a process group
*/
static void
pgdelete(struct pgrp *pgrp)
{
struct session *savesess;
struct tty *tp;
sx_assert(&proctree_lock, SX_XLOCKED);
PGRP_LOCK_ASSERT(pgrp, MA_NOTOWNED);
SESS_LOCK_ASSERT(pgrp->pg_session, MA_NOTOWNED);
/*
* Reset any sigio structures pointing to us as a result of
* F_SETOWN with our pgid.
*/
funsetownlst(&pgrp->pg_sigiolst);
PGRP_LOCK(pgrp);
tp = pgrp->pg_session->s_ttyp;
LIST_REMOVE(pgrp, pg_hash);
savesess = pgrp->pg_session;
PGRP_UNLOCK(pgrp);
/* Remove the reference to the pgrp before deallocating it. */
if (tp != NULL) {
tty_lock(tp);
tty_rel_pgrp(tp, pgrp);
}
proc_id_clear(PROC_ID_GROUP, pgrp->pg_id);
mtx_destroy(&pgrp->pg_mtx);
free(pgrp, M_PGRP);
sess_release(savesess);
}
static void
pgadjustjobc(struct pgrp *pgrp, int entering)
{
PGRP_LOCK(pgrp);
if (entering)
pgrp->pg_jobc++;
else {
--pgrp->pg_jobc;
if (pgrp->pg_jobc == 0)
orphanpg(pgrp);
}
PGRP_UNLOCK(pgrp);
}
/*
* Adjust pgrp jobc counters when specified process changes process group.
* We count the number of processes in each process group that "qualify"
* the group for terminal job control (those with a parent in a different
* process group of the same session). If that count reaches zero, the
* process group becomes orphaned. Check both the specified process'
* process group and that of its children.
* entering == 0 => p is leaving specified group.
* entering == 1 => p is entering specified group.
*/
void
fixjobc(struct proc *p, struct pgrp *pgrp, int entering)
{
struct pgrp *hispgrp;
struct session *mysession;
struct proc *q;
sx_assert(&proctree_lock, SX_LOCKED);
PROC_LOCK_ASSERT(p, MA_NOTOWNED);
PGRP_LOCK_ASSERT(pgrp, MA_NOTOWNED);
SESS_LOCK_ASSERT(pgrp->pg_session, MA_NOTOWNED);
/*
* Check p's parent to see whether p qualifies its own process
* group; if so, adjust count for p's process group.
*/
mysession = pgrp->pg_session;
if ((hispgrp = p->p_pptr->p_pgrp) != pgrp &&
hispgrp->pg_session == mysession)
pgadjustjobc(pgrp, entering);
/*
* Check this process' children to see whether they qualify
* their process groups; if so, adjust counts for children's
* process groups.
*/
LIST_FOREACH(q, &p->p_children, p_sibling) {
hispgrp = q->p_pgrp;
if (hispgrp == pgrp ||
hispgrp->pg_session != mysession)
continue;
if (q->p_state == PRS_ZOMBIE)
continue;
pgadjustjobc(hispgrp, entering);
}
}
void
killjobc(void)
{
struct session *sp;
struct tty *tp;
struct proc *p;
struct vnode *ttyvp;
p = curproc;
MPASS(p->p_flag & P_WEXIT);
/*
* Do a quick check to see if there is anything to do with the
* proctree_lock held. pgrp and LIST_EMPTY checks are for fixjobc().
*/
PROC_LOCK(p);
if (!SESS_LEADER(p) &&
(p->p_pgrp == p->p_pptr->p_pgrp) &&
LIST_EMPTY(&p->p_children)) {
PROC_UNLOCK(p);
return;
}
PROC_UNLOCK(p);
sx_xlock(&proctree_lock);
if (SESS_LEADER(p)) {
sp = p->p_session;
/*
* s_ttyp is not zero'd; we use this to indicate that
* the session once had a controlling terminal. (for
* logging and informational purposes)
*/
SESS_LOCK(sp);
ttyvp = sp->s_ttyvp;
tp = sp->s_ttyp;
sp->s_ttyvp = NULL;
sp->s_ttydp = NULL;
sp->s_leader = NULL;
SESS_UNLOCK(sp);
/*
* Signal foreground pgrp and revoke access to
* controlling terminal if it has not been revoked
* already.
*
* Because the TTY may have been revoked in the mean
* time and could already have a new session associated
* with it, make sure we don't send a SIGHUP to a
* foreground process group that does not belong to this
* session.
*/
if (tp != NULL) {
tty_lock(tp);
if (tp->t_session == sp)
tty_signal_pgrp(tp, SIGHUP);
tty_unlock(tp);
}
if (ttyvp != NULL) {
sx_xunlock(&proctree_lock);
if (vn_lock(ttyvp, LK_EXCLUSIVE) == 0) {
VOP_REVOKE(ttyvp, REVOKEALL);
VOP_UNLOCK(ttyvp, 0);
}
vrele(ttyvp);
sx_xlock(&proctree_lock);
}
}
fixjobc(p, p->p_pgrp, 0);
sx_xunlock(&proctree_lock);
}
/*
* A process group has become orphaned;
* if there are any stopped processes in the group,
* hang-up all process in that group.
*/
static void
orphanpg(struct pgrp *pg)
{
struct proc *p;
PGRP_LOCK_ASSERT(pg, MA_OWNED);
LIST_FOREACH(p, &pg->pg_members, p_pglist) {
PROC_LOCK(p);
if (P_SHOULDSTOP(p) == P_STOPPED_SIG) {
PROC_UNLOCK(p);
LIST_FOREACH(p, &pg->pg_members, p_pglist) {
PROC_LOCK(p);
kern_psignal(p, SIGHUP);
kern_psignal(p, SIGCONT);
PROC_UNLOCK(p);
}
return;
}
PROC_UNLOCK(p);
}
}
void
sess_hold(struct session *s)
{
refcount_acquire(&s->s_count);
}
void
sess_release(struct session *s)
{
if (refcount_release(&s->s_count)) {
if (s->s_ttyp != NULL) {
tty_lock(s->s_ttyp);
tty_rel_sess(s->s_ttyp, s);
}
proc_id_clear(PROC_ID_SESSION, s->s_sid);
mtx_destroy(&s->s_mtx);
free(s, M_SESSION);
}
}
#ifdef DDB
DB_SHOW_COMMAND(pgrpdump, pgrpdump)
{
struct pgrp *pgrp;
struct proc *p;
int i;
for (i = 0; i <= pgrphash; i++) {
if (!LIST_EMPTY(&pgrphashtbl[i])) {
printf("\tindx %d\n", i);
LIST_FOREACH(pgrp, &pgrphashtbl[i], pg_hash) {
printf(
"\tpgrp %p, pgid %ld, sess %p, sesscnt %d, mem %p\n",
(void *)pgrp, (long)pgrp->pg_id,
(void *)pgrp->pg_session,
pgrp->pg_session->s_count,
(void *)LIST_FIRST(&pgrp->pg_members));
LIST_FOREACH(p, &pgrp->pg_members, p_pglist) {
printf("\t\tpid %ld addr %p pgrp %p\n",
(long)p->p_pid, (void *)p,
(void *)p->p_pgrp);
}
}
}
}
}
#endif /* DDB */
/*
* Calculate the kinfo_proc members which contain process-wide
* informations.
* Must be called with the target process locked.
*/
static void
fill_kinfo_aggregate(struct proc *p, struct kinfo_proc *kp)
{
struct thread *td;
PROC_LOCK_ASSERT(p, MA_OWNED);
kp->ki_estcpu = 0;
kp->ki_pctcpu = 0;
FOREACH_THREAD_IN_PROC(p, td) {
thread_lock(td);
kp->ki_pctcpu += sched_pctcpu(td);
kp->ki_estcpu += sched_estcpu(td);
thread_unlock(td);
}
}
/*
* Clear kinfo_proc and fill in any information that is common
* to all threads in the process.
* Must be called with the target process locked.
*/
static void
fill_kinfo_proc_only(struct proc *p, struct kinfo_proc *kp)
{
struct thread *td0;
struct tty *tp;
struct session *sp;
struct ucred *cred;
struct sigacts *ps;
struct timeval boottime;
PROC_LOCK_ASSERT(p, MA_OWNED);
bzero(kp, sizeof(*kp));
kp->ki_structsize = sizeof(*kp);
kp->ki_paddr = p;
kp->ki_addr =/* p->p_addr; */0; /* XXX */
kp->ki_args = p->p_args;
kp->ki_textvp = p->p_textvp;
#ifdef KTRACE
kp->ki_tracep = p->p_tracevp;
kp->ki_traceflag = p->p_traceflag;
#endif
kp->ki_fd = p->p_fd;
kp->ki_vmspace = p->p_vmspace;
kp->ki_flag = p->p_flag;
kp->ki_flag2 = p->p_flag2;
cred = p->p_ucred;
if (cred) {
kp->ki_uid = cred->cr_uid;
kp->ki_ruid = cred->cr_ruid;
kp->ki_svuid = cred->cr_svuid;
kp->ki_cr_flags = 0;
if (cred->cr_flags & CRED_FLAG_CAPMODE)
kp->ki_cr_flags |= KI_CRF_CAPABILITY_MODE;
/* XXX bde doesn't like KI_NGROUPS */
if (cred->cr_ngroups > KI_NGROUPS) {
kp->ki_ngroups = KI_NGROUPS;
kp->ki_cr_flags |= KI_CRF_GRP_OVERFLOW;
} else
kp->ki_ngroups = cred->cr_ngroups;
bcopy(cred->cr_groups, kp->ki_groups,
kp->ki_ngroups * sizeof(gid_t));
kp->ki_rgid = cred->cr_rgid;
kp->ki_svgid = cred->cr_svgid;
/* If jailed(cred), emulate the old P_JAILED flag. */
if (jailed(cred)) {
kp->ki_flag |= P_JAILED;
/* If inside the jail, use 0 as a jail ID. */
if (cred->cr_prison != curthread->td_ucred->cr_prison)
kp->ki_jid = cred->cr_prison->pr_id;
}
strlcpy(kp->ki_loginclass, cred->cr_loginclass->lc_name,
sizeof(kp->ki_loginclass));
}
ps = p->p_sigacts;
if (ps) {
mtx_lock(&ps->ps_mtx);
kp->ki_sigignore = ps->ps_sigignore;
kp->ki_sigcatch = ps->ps_sigcatch;
mtx_unlock(&ps->ps_mtx);
}
if (p->p_state != PRS_NEW &&
p->p_state != PRS_ZOMBIE &&
p->p_vmspace != NULL) {
struct vmspace *vm = p->p_vmspace;
kp->ki_size = vm->vm_map.size;
kp->ki_rssize = vmspace_resident_count(vm); /*XXX*/
FOREACH_THREAD_IN_PROC(p, td0) {
if (!TD_IS_SWAPPED(td0))
kp->ki_rssize += td0->td_kstack_pages;
}
kp->ki_swrss = vm->vm_swrss;
kp->ki_tsize = vm->vm_tsize;
kp->ki_dsize = vm->vm_dsize;
kp->ki_ssize = vm->vm_ssize;
} else if (p->p_state == PRS_ZOMBIE)
kp->ki_stat = SZOMB;
if (kp->ki_flag & P_INMEM)
kp->ki_sflag = PS_INMEM;
else
kp->ki_sflag = 0;
/* Calculate legacy swtime as seconds since 'swtick'. */
kp->ki_swtime = (ticks - p->p_swtick) / hz;
kp->ki_pid = p->p_pid;
kp->ki_nice = p->p_nice;
kp->ki_fibnum = p->p_fibnum;
kp->ki_start = p->p_stats->p_start;
getboottime(&boottime);
timevaladd(&kp->ki_start, &boottime);
PROC_STATLOCK(p);
rufetch(p, &kp->ki_rusage);
kp->ki_runtime = cputick2usec(p->p_rux.rux_runtime);
calcru(p, &kp->ki_rusage.ru_utime, &kp->ki_rusage.ru_stime);
PROC_STATUNLOCK(p);
calccru(p, &kp->ki_childutime, &kp->ki_childstime);
/* Some callers want child times in a single value. */
kp->ki_childtime = kp->ki_childstime;
timevaladd(&kp->ki_childtime, &kp->ki_childutime);
FOREACH_THREAD_IN_PROC(p, td0)
kp->ki_cow += td0->td_cow;
tp = NULL;
if (p->p_pgrp) {
kp->ki_pgid = p->p_pgrp->pg_id;
kp->ki_jobc = p->p_pgrp->pg_jobc;
sp = p->p_pgrp->pg_session;
if (sp != NULL) {
kp->ki_sid = sp->s_sid;
SESS_LOCK(sp);
strlcpy(kp->ki_login, sp->s_login,
sizeof(kp->ki_login));
if (sp->s_ttyvp)
kp->ki_kiflag |= KI_CTTY;
if (SESS_LEADER(p))
kp->ki_kiflag |= KI_SLEADER;
/* XXX proctree_lock */
tp = sp->s_ttyp;
SESS_UNLOCK(sp);
}
}
if ((p->p_flag & P_CONTROLT) && tp != NULL) {
kp->ki_tdev = tty_udev(tp);
kp->ki_tdev_freebsd11 = kp->ki_tdev; /* truncate */
kp->ki_tpgid = tp->t_pgrp ? tp->t_pgrp->pg_id : NO_PID;
if (tp->t_session)
kp->ki_tsid = tp->t_session->s_sid;
} else {
kp->ki_tdev = NODEV;
kp->ki_tdev_freebsd11 = kp->ki_tdev; /* truncate */
}
if (p->p_comm[0] != '\0')
strlcpy(kp->ki_comm, p->p_comm, sizeof(kp->ki_comm));
if (p->p_sysent && p->p_sysent->sv_name != NULL &&
p->p_sysent->sv_name[0] != '\0')
strlcpy(kp->ki_emul, p->p_sysent->sv_name, sizeof(kp->ki_emul));
kp->ki_siglist = p->p_siglist;
kp->ki_xstat = KW_EXITCODE(p->p_xexit, p->p_xsig);
kp->ki_acflag = p->p_acflag;
kp->ki_lock = p->p_lock;
if (p->p_pptr) {
kp->ki_ppid = p->p_oppid;
if (p->p_flag & P_TRACED)
kp->ki_tracer = p->p_pptr->p_pid;
}
}
/*
* Fill in information that is thread specific. Must be called with
* target process locked. If 'preferthread' is set, overwrite certain
* process-related fields that are maintained for both threads and
* processes.
*/
static void
fill_kinfo_thread(struct thread *td, struct kinfo_proc *kp, int preferthread)
{
struct proc *p;
p = td->td_proc;
kp->ki_tdaddr = td;
PROC_LOCK_ASSERT(p, MA_OWNED);
if (preferthread)
PROC_STATLOCK(p);
thread_lock(td);
if (td->td_wmesg != NULL)
strlcpy(kp->ki_wmesg, td->td_wmesg, sizeof(kp->ki_wmesg));
else
bzero(kp->ki_wmesg, sizeof(kp->ki_wmesg));
if (strlcpy(kp->ki_tdname, td->td_name, sizeof(kp->ki_tdname)) >=
sizeof(kp->ki_tdname)) {
strlcpy(kp->ki_moretdname,
td->td_name + sizeof(kp->ki_tdname) - 1,
sizeof(kp->ki_moretdname));
} else {
bzero(kp->ki_moretdname, sizeof(kp->ki_moretdname));
}
if (TD_ON_LOCK(td)) {
kp->ki_kiflag |= KI_LOCKBLOCK;
strlcpy(kp->ki_lockname, td->td_lockname,
sizeof(kp->ki_lockname));
} else {
kp->ki_kiflag &= ~KI_LOCKBLOCK;
bzero(kp->ki_lockname, sizeof(kp->ki_lockname));
}
if (p->p_state == PRS_NORMAL) { /* approximate. */
if (TD_ON_RUNQ(td) ||
TD_CAN_RUN(td) ||
TD_IS_RUNNING(td)) {
kp->ki_stat = SRUN;
} else if (P_SHOULDSTOP(p)) {
kp->ki_stat = SSTOP;
} else if (TD_IS_SLEEPING(td)) {
kp->ki_stat = SSLEEP;
} else if (TD_ON_LOCK(td)) {
kp->ki_stat = SLOCK;
} else {
kp->ki_stat = SWAIT;
}
} else if (p->p_state == PRS_ZOMBIE) {
kp->ki_stat = SZOMB;
} else {
kp->ki_stat = SIDL;
}
/* Things in the thread */
kp->ki_wchan = td->td_wchan;
kp->ki_pri.pri_level = td->td_priority;
kp->ki_pri.pri_native = td->td_base_pri;
/*
* Note: legacy fields; clamp at the old NOCPU value and/or
* the maximum u_char CPU value.
*/
if (td->td_lastcpu == NOCPU)
kp->ki_lastcpu_old = NOCPU_OLD;
else if (td->td_lastcpu > MAXCPU_OLD)
kp->ki_lastcpu_old = MAXCPU_OLD;
else
kp->ki_lastcpu_old = td->td_lastcpu;
if (td->td_oncpu == NOCPU)
kp->ki_oncpu_old = NOCPU_OLD;
else if (td->td_oncpu > MAXCPU_OLD)
kp->ki_oncpu_old = MAXCPU_OLD;
else
kp->ki_oncpu_old = td->td_oncpu;
kp->ki_lastcpu = td->td_lastcpu;
kp->ki_oncpu = td->td_oncpu;
kp->ki_tdflags = td->td_flags;
kp->ki_tid = td->td_tid;
kp->ki_numthreads = p->p_numthreads;
kp->ki_pcb = td->td_pcb;
kp->ki_kstack = (void *)td->td_kstack;
kp->ki_slptime = (ticks - td->td_slptick) / hz;
kp->ki_pri.pri_class = td->td_pri_class;
kp->ki_pri.pri_user = td->td_user_pri;
if (preferthread) {
rufetchtd(td, &kp->ki_rusage);
kp->ki_runtime = cputick2usec(td->td_rux.rux_runtime);
kp->ki_pctcpu = sched_pctcpu(td);
kp->ki_estcpu = sched_estcpu(td);
kp->ki_cow = td->td_cow;
}
/* We can't get this anymore but ps etc never used it anyway. */
kp->ki_rqindex = 0;
if (preferthread)
kp->ki_siglist = td->td_siglist;
kp->ki_sigmask = td->td_sigmask;
thread_unlock(td);
if (preferthread)
PROC_STATUNLOCK(p);
}
/*
* Fill in a kinfo_proc structure for the specified process.
* Must be called with the target process locked.
*/
void
fill_kinfo_proc(struct proc *p, struct kinfo_proc *kp)
{
MPASS(FIRST_THREAD_IN_PROC(p) != NULL);
fill_kinfo_proc_only(p, kp);
fill_kinfo_thread(FIRST_THREAD_IN_PROC(p), kp, 0);
fill_kinfo_aggregate(p, kp);
}
struct pstats *
pstats_alloc(void)
{
return (malloc(sizeof(struct pstats), M_SUBPROC, M_ZERO|M_WAITOK));
}
/*
* Copy parts of p_stats; zero the rest of p_stats (statistics).
*/
void
pstats_fork(struct pstats *src, struct pstats *dst)
{
bzero(&dst->pstat_startzero,
__rangeof(struct pstats, pstat_startzero, pstat_endzero));
bcopy(&src->pstat_startcopy, &dst->pstat_startcopy,
__rangeof(struct pstats, pstat_startcopy, pstat_endcopy));
}
void
pstats_free(struct pstats *ps)
{
free(ps, M_SUBPROC);
}
#ifdef COMPAT_FREEBSD32
/*
* This function is typically used to copy out the kernel address, so
* it can be replaced by assignment of zero.
*/
static inline uint32_t
-ptr32_trim(void *ptr)
+ptr32_trim(const void *ptr)
{
uintptr_t uptr;
uptr = (uintptr_t)ptr;
return ((uptr > UINT_MAX) ? 0 : uptr);
}
#define PTRTRIM_CP(src,dst,fld) \
do { (dst).fld = ptr32_trim((src).fld); } while (0)
static void
freebsd32_kinfo_proc_out(const struct kinfo_proc *ki, struct kinfo_proc32 *ki32)
{
int i;
bzero(ki32, sizeof(struct kinfo_proc32));
ki32->ki_structsize = sizeof(struct kinfo_proc32);
CP(*ki, *ki32, ki_layout);
PTRTRIM_CP(*ki, *ki32, ki_args);
PTRTRIM_CP(*ki, *ki32, ki_paddr);
PTRTRIM_CP(*ki, *ki32, ki_addr);
PTRTRIM_CP(*ki, *ki32, ki_tracep);
PTRTRIM_CP(*ki, *ki32, ki_textvp);
PTRTRIM_CP(*ki, *ki32, ki_fd);
PTRTRIM_CP(*ki, *ki32, ki_vmspace);
PTRTRIM_CP(*ki, *ki32, ki_wchan);
CP(*ki, *ki32, ki_pid);
CP(*ki, *ki32, ki_ppid);
CP(*ki, *ki32, ki_pgid);
CP(*ki, *ki32, ki_tpgid);
CP(*ki, *ki32, ki_sid);
CP(*ki, *ki32, ki_tsid);
CP(*ki, *ki32, ki_jobc);
CP(*ki, *ki32, ki_tdev);
CP(*ki, *ki32, ki_tdev_freebsd11);
CP(*ki, *ki32, ki_siglist);
CP(*ki, *ki32, ki_sigmask);
CP(*ki, *ki32, ki_sigignore);
CP(*ki, *ki32, ki_sigcatch);
CP(*ki, *ki32, ki_uid);
CP(*ki, *ki32, ki_ruid);
CP(*ki, *ki32, ki_svuid);
CP(*ki, *ki32, ki_rgid);
CP(*ki, *ki32, ki_svgid);
CP(*ki, *ki32, ki_ngroups);
for (i = 0; i < KI_NGROUPS; i++)
CP(*ki, *ki32, ki_groups[i]);
CP(*ki, *ki32, ki_size);
CP(*ki, *ki32, ki_rssize);
CP(*ki, *ki32, ki_swrss);
CP(*ki, *ki32, ki_tsize);
CP(*ki, *ki32, ki_dsize);
CP(*ki, *ki32, ki_ssize);
CP(*ki, *ki32, ki_xstat);
CP(*ki, *ki32, ki_acflag);
CP(*ki, *ki32, ki_pctcpu);
CP(*ki, *ki32, ki_estcpu);
CP(*ki, *ki32, ki_slptime);
CP(*ki, *ki32, ki_swtime);
CP(*ki, *ki32, ki_cow);
CP(*ki, *ki32, ki_runtime);
TV_CP(*ki, *ki32, ki_start);
TV_CP(*ki, *ki32, ki_childtime);
CP(*ki, *ki32, ki_flag);
CP(*ki, *ki32, ki_kiflag);
CP(*ki, *ki32, ki_traceflag);
CP(*ki, *ki32, ki_stat);
CP(*ki, *ki32, ki_nice);
CP(*ki, *ki32, ki_lock);
CP(*ki, *ki32, ki_rqindex);
CP(*ki, *ki32, ki_oncpu);
CP(*ki, *ki32, ki_lastcpu);
/* XXX TODO: wrap cpu value as appropriate */
CP(*ki, *ki32, ki_oncpu_old);
CP(*ki, *ki32, ki_lastcpu_old);
bcopy(ki->ki_tdname, ki32->ki_tdname, TDNAMLEN + 1);
bcopy(ki->ki_wmesg, ki32->ki_wmesg, WMESGLEN + 1);
bcopy(ki->ki_login, ki32->ki_login, LOGNAMELEN + 1);
bcopy(ki->ki_lockname, ki32->ki_lockname, LOCKNAMELEN + 1);
bcopy(ki->ki_comm, ki32->ki_comm, COMMLEN + 1);
bcopy(ki->ki_emul, ki32->ki_emul, KI_EMULNAMELEN + 1);
bcopy(ki->ki_loginclass, ki32->ki_loginclass, LOGINCLASSLEN + 1);
bcopy(ki->ki_moretdname, ki32->ki_moretdname, MAXCOMLEN - TDNAMLEN + 1);
CP(*ki, *ki32, ki_tracer);
CP(*ki, *ki32, ki_flag2);
CP(*ki, *ki32, ki_fibnum);
CP(*ki, *ki32, ki_cr_flags);
CP(*ki, *ki32, ki_jid);
CP(*ki, *ki32, ki_numthreads);
CP(*ki, *ki32, ki_tid);
CP(*ki, *ki32, ki_pri);
freebsd32_rusage_out(&ki->ki_rusage, &ki32->ki_rusage);
freebsd32_rusage_out(&ki->ki_rusage_ch, &ki32->ki_rusage_ch);
PTRTRIM_CP(*ki, *ki32, ki_pcb);
PTRTRIM_CP(*ki, *ki32, ki_kstack);
PTRTRIM_CP(*ki, *ki32, ki_udata);
PTRTRIM_CP(*ki, *ki32, ki_tdaddr);
CP(*ki, *ki32, ki_sflag);
CP(*ki, *ki32, ki_tdflags);
}
#endif
static ssize_t
kern_proc_out_size(struct proc *p, int flags)
{
ssize_t size = 0;
PROC_LOCK_ASSERT(p, MA_OWNED);
if ((flags & KERN_PROC_NOTHREADS) != 0) {
#ifdef COMPAT_FREEBSD32
if ((flags & KERN_PROC_MASK32) != 0) {
size += sizeof(struct kinfo_proc32);
} else
#endif
size += sizeof(struct kinfo_proc);
} else {
#ifdef COMPAT_FREEBSD32
if ((flags & KERN_PROC_MASK32) != 0)
size += sizeof(struct kinfo_proc32) * p->p_numthreads;
else
#endif
size += sizeof(struct kinfo_proc) * p->p_numthreads;
}
PROC_UNLOCK(p);
return (size);
}
int
kern_proc_out(struct proc *p, struct sbuf *sb, int flags)
{
struct thread *td;
struct kinfo_proc ki;
#ifdef COMPAT_FREEBSD32
struct kinfo_proc32 ki32;
#endif
int error;
PROC_LOCK_ASSERT(p, MA_OWNED);
MPASS(FIRST_THREAD_IN_PROC(p) != NULL);
error = 0;
fill_kinfo_proc(p, &ki);
if ((flags & KERN_PROC_NOTHREADS) != 0) {
#ifdef COMPAT_FREEBSD32
if ((flags & KERN_PROC_MASK32) != 0) {
freebsd32_kinfo_proc_out(&ki, &ki32);
if (sbuf_bcat(sb, &ki32, sizeof(ki32)) != 0)
error = ENOMEM;
} else
#endif
if (sbuf_bcat(sb, &ki, sizeof(ki)) != 0)
error = ENOMEM;
} else {
FOREACH_THREAD_IN_PROC(p, td) {
fill_kinfo_thread(td, &ki, 1);
#ifdef COMPAT_FREEBSD32
if ((flags & KERN_PROC_MASK32) != 0) {
freebsd32_kinfo_proc_out(&ki, &ki32);
if (sbuf_bcat(sb, &ki32, sizeof(ki32)) != 0)
error = ENOMEM;
} else
#endif
if (sbuf_bcat(sb, &ki, sizeof(ki)) != 0)
error = ENOMEM;
if (error != 0)
break;
}
}
PROC_UNLOCK(p);
return (error);
}
static int
sysctl_out_proc(struct proc *p, struct sysctl_req *req, int flags)
{
struct sbuf sb;
struct kinfo_proc ki;
int error, error2;
if (req->oldptr == NULL)
return (SYSCTL_OUT(req, 0, kern_proc_out_size(p, flags)));
sbuf_new_for_sysctl(&sb, (char *)&ki, sizeof(ki), req);
sbuf_clear_flags(&sb, SBUF_INCLUDENUL);
error = kern_proc_out(p, &sb, flags);
error2 = sbuf_finish(&sb);
sbuf_delete(&sb);
if (error != 0)
return (error);
else if (error2 != 0)
return (error2);
return (0);
}
int
proc_iterate(int (*cb)(struct proc *, void *), void *cbarg)
{
struct proc *p;
int error, i, j;
for (i = 0; i < pidhashlock + 1; i++) {
sx_slock(&pidhashtbl_lock[i]);
for (j = i; j <= pidhash; j += pidhashlock + 1) {
LIST_FOREACH(p, &pidhashtbl[j], p_hash) {
if (p->p_state == PRS_NEW)
continue;
error = cb(p, cbarg);
PROC_LOCK_ASSERT(p, MA_NOTOWNED);
if (error != 0) {
sx_sunlock(&pidhashtbl_lock[i]);
return (error);
}
}
}
sx_sunlock(&pidhashtbl_lock[i]);
}
return (0);
}
struct kern_proc_out_args {
struct sysctl_req *req;
int flags;
int oid_number;
int *name;
};
static int
sysctl_kern_proc_iterate(struct proc *p, void *origarg)
{
struct kern_proc_out_args *arg = origarg;
int *name = arg->name;
int oid_number = arg->oid_number;
int flags = arg->flags;
struct sysctl_req *req = arg->req;
int error = 0;
PROC_LOCK(p);
KASSERT(p->p_ucred != NULL,
("process credential is NULL for non-NEW proc"));
/*
* Show a user only appropriate processes.
*/
if (p_cansee(curthread, p))
goto skip;
/*
* TODO - make more efficient (see notes below).
* do by session.
*/
switch (oid_number) {
case KERN_PROC_GID:
if (p->p_ucred->cr_gid != (gid_t)name[0])
goto skip;
break;
case KERN_PROC_PGRP:
/* could do this by traversing pgrp */
if (p->p_pgrp == NULL ||
p->p_pgrp->pg_id != (pid_t)name[0])
goto skip;
break;
case KERN_PROC_RGID:
if (p->p_ucred->cr_rgid != (gid_t)name[0])
goto skip;
break;
case KERN_PROC_SESSION:
if (p->p_session == NULL ||
p->p_session->s_sid != (pid_t)name[0])
goto skip;
break;
case KERN_PROC_TTY:
if ((p->p_flag & P_CONTROLT) == 0 ||
p->p_session == NULL)
goto skip;
/* XXX proctree_lock */
SESS_LOCK(p->p_session);
if (p->p_session->s_ttyp == NULL ||
tty_udev(p->p_session->s_ttyp) !=
(dev_t)name[0]) {
SESS_UNLOCK(p->p_session);
goto skip;
}
SESS_UNLOCK(p->p_session);
break;
case KERN_PROC_UID:
if (p->p_ucred->cr_uid != (uid_t)name[0])
goto skip;
break;
case KERN_PROC_RUID:
if (p->p_ucred->cr_ruid != (uid_t)name[0])
goto skip;
break;
case KERN_PROC_PROC:
break;
default:
break;
}
error = sysctl_out_proc(p, req, flags);
PROC_LOCK_ASSERT(p, MA_NOTOWNED);
return (error);
skip:
PROC_UNLOCK(p);
return (0);
}
static int
sysctl_kern_proc(SYSCTL_HANDLER_ARGS)
{
struct kern_proc_out_args iterarg;
int *name = (int *)arg1;
u_int namelen = arg2;
struct proc *p;
int flags, oid_number;
int error = 0;
oid_number = oidp->oid_number;
if (oid_number != KERN_PROC_ALL &&
(oid_number & KERN_PROC_INC_THREAD) == 0)
flags = KERN_PROC_NOTHREADS;
else {
flags = 0;
oid_number &= ~KERN_PROC_INC_THREAD;
}
#ifdef COMPAT_FREEBSD32
if (req->flags & SCTL_MASK32)
flags |= KERN_PROC_MASK32;
#endif
if (oid_number == KERN_PROC_PID) {
if (namelen != 1)
return (EINVAL);
error = sysctl_wire_old_buffer(req, 0);
if (error)
return (error);
error = pget((pid_t)name[0], PGET_CANSEE, &p);
if (error == 0)
error = sysctl_out_proc(p, req, flags);
return (error);
}
switch (oid_number) {
case KERN_PROC_ALL:
if (namelen != 0)
return (EINVAL);
break;
case KERN_PROC_PROC:
if (namelen != 0 && namelen != 1)
return (EINVAL);
break;
default:
if (namelen != 1)
return (EINVAL);
break;
}
if (req->oldptr == NULL) {
/* overestimate by 5 procs */
error = SYSCTL_OUT(req, 0, sizeof (struct kinfo_proc) * 5);
if (error)
return (error);
} else {
error = sysctl_wire_old_buffer(req, 0);
if (error != 0)
return (error);
}
iterarg.flags = flags;
iterarg.oid_number = oid_number;
iterarg.req = req;
iterarg.name = name;
error = proc_iterate(sysctl_kern_proc_iterate, &iterarg);
return (error);
}
struct pargs *
pargs_alloc(int len)
{
struct pargs *pa;
pa = malloc(sizeof(struct pargs) + len, M_PARGS,
M_WAITOK);
refcount_init(&pa->ar_ref, 1);
pa->ar_length = len;
return (pa);
}
static void
pargs_free(struct pargs *pa)
{
free(pa, M_PARGS);
}
void
pargs_hold(struct pargs *pa)
{
if (pa == NULL)
return;
refcount_acquire(&pa->ar_ref);
}
void
pargs_drop(struct pargs *pa)
{
if (pa == NULL)
return;
if (refcount_release(&pa->ar_ref))
pargs_free(pa);
}
static int
proc_read_string(struct thread *td, struct proc *p, const char *sptr, char *buf,
size_t len)
{
ssize_t n;
/*
* This may return a short read if the string is shorter than the chunk
* and is aligned at the end of the page, and the following page is not
* mapped.
*/
n = proc_readmem(td, p, (vm_offset_t)sptr, buf, len);
if (n <= 0)
return (ENOMEM);
return (0);
}
#define PROC_AUXV_MAX 256 /* Safety limit on auxv size. */
enum proc_vector_type {
PROC_ARG,
PROC_ENV,
PROC_AUX,
};
#ifdef COMPAT_FREEBSD32
static int
get_proc_vector32(struct thread *td, struct proc *p, char ***proc_vectorp,
size_t *vsizep, enum proc_vector_type type)
{
struct freebsd32_ps_strings pss;
Elf32_Auxinfo aux;
vm_offset_t vptr, ptr;
uint32_t *proc_vector32;
char **proc_vector;
size_t vsize, size;
int i, error;
error = 0;
if (proc_readmem(td, p, (vm_offset_t)p->p_sysent->sv_psstrings, &pss,
sizeof(pss)) != sizeof(pss))
return (ENOMEM);
switch (type) {
case PROC_ARG:
vptr = (vm_offset_t)PTRIN(pss.ps_argvstr);
vsize = pss.ps_nargvstr;
if (vsize > ARG_MAX)
return (ENOEXEC);
size = vsize * sizeof(int32_t);
break;
case PROC_ENV:
vptr = (vm_offset_t)PTRIN(pss.ps_envstr);
vsize = pss.ps_nenvstr;
if (vsize > ARG_MAX)
return (ENOEXEC);
size = vsize * sizeof(int32_t);
break;
case PROC_AUX:
vptr = (vm_offset_t)PTRIN(pss.ps_envstr) +
(pss.ps_nenvstr + 1) * sizeof(int32_t);
if (vptr % 4 != 0)
return (ENOEXEC);
for (ptr = vptr, i = 0; i < PROC_AUXV_MAX; i++) {
if (proc_readmem(td, p, ptr, &aux, sizeof(aux)) !=
sizeof(aux))
return (ENOMEM);
if (aux.a_type == AT_NULL)
break;
ptr += sizeof(aux);
}
if (aux.a_type != AT_NULL)
return (ENOEXEC);
vsize = i + 1;
size = vsize * sizeof(aux);
break;
default:
KASSERT(0, ("Wrong proc vector type: %d", type));
return (EINVAL);
}
proc_vector32 = malloc(size, M_TEMP, M_WAITOK);
if (proc_readmem(td, p, vptr, proc_vector32, size) != size) {
error = ENOMEM;
goto done;
}
if (type == PROC_AUX) {
*proc_vectorp = (char **)proc_vector32;
*vsizep = vsize;
return (0);
}
proc_vector = malloc(vsize * sizeof(char *), M_TEMP, M_WAITOK);
for (i = 0; i < (int)vsize; i++)
proc_vector[i] = PTRIN(proc_vector32[i]);
*proc_vectorp = proc_vector;
*vsizep = vsize;
done:
free(proc_vector32, M_TEMP);
return (error);
}
#endif
static int
get_proc_vector(struct thread *td, struct proc *p, char ***proc_vectorp,
size_t *vsizep, enum proc_vector_type type)
{
struct ps_strings pss;
Elf_Auxinfo aux;
vm_offset_t vptr, ptr;
char **proc_vector;
size_t vsize, size;
int i;
#ifdef COMPAT_FREEBSD32
if (SV_PROC_FLAG(p, SV_ILP32) != 0)
return (get_proc_vector32(td, p, proc_vectorp, vsizep, type));
#endif
if (proc_readmem(td, p, (vm_offset_t)p->p_sysent->sv_psstrings, &pss,
sizeof(pss)) != sizeof(pss))
return (ENOMEM);
switch (type) {
case PROC_ARG:
vptr = (vm_offset_t)pss.ps_argvstr;
vsize = pss.ps_nargvstr;
if (vsize > ARG_MAX)
return (ENOEXEC);
size = vsize * sizeof(char *);
break;
case PROC_ENV:
vptr = (vm_offset_t)pss.ps_envstr;
vsize = pss.ps_nenvstr;
if (vsize > ARG_MAX)
return (ENOEXEC);
size = vsize * sizeof(char *);
break;
case PROC_AUX:
/*
* The aux array is just above env array on the stack. Check
* that the address is naturally aligned.
*/
vptr = (vm_offset_t)pss.ps_envstr + (pss.ps_nenvstr + 1)
* sizeof(char *);
#if __ELF_WORD_SIZE == 64
if (vptr % sizeof(uint64_t) != 0)
#else
if (vptr % sizeof(uint32_t) != 0)
#endif
return (ENOEXEC);
/*
* We count the array size reading the aux vectors from the
* stack until AT_NULL vector is returned. So (to keep the code
* simple) we read the process stack twice: the first time here
* to find the size and the second time when copying the vectors
* to the allocated proc_vector.
*/
for (ptr = vptr, i = 0; i < PROC_AUXV_MAX; i++) {
if (proc_readmem(td, p, ptr, &aux, sizeof(aux)) !=
sizeof(aux))
return (ENOMEM);
if (aux.a_type == AT_NULL)
break;
ptr += sizeof(aux);
}
/*
* If the PROC_AUXV_MAX entries are iterated over, and we have
* not reached AT_NULL, it is most likely we are reading wrong
* data: either the process doesn't have auxv array or data has
* been modified. Return the error in this case.
*/
if (aux.a_type != AT_NULL)
return (ENOEXEC);
vsize = i + 1;
size = vsize * sizeof(aux);
break;
default:
KASSERT(0, ("Wrong proc vector type: %d", type));
return (EINVAL); /* In case we are built without INVARIANTS. */
}
proc_vector = malloc(size, M_TEMP, M_WAITOK);
if (proc_readmem(td, p, vptr, proc_vector, size) != size) {
free(proc_vector, M_TEMP);
return (ENOMEM);
}
*proc_vectorp = proc_vector;
*vsizep = vsize;
return (0);
}
#define GET_PS_STRINGS_CHUNK_SZ 256 /* Chunk size (bytes) for ps_strings operations. */
static int
get_ps_strings(struct thread *td, struct proc *p, struct sbuf *sb,
enum proc_vector_type type)
{
size_t done, len, nchr, vsize;
int error, i;
char **proc_vector, *sptr;
char pss_string[GET_PS_STRINGS_CHUNK_SZ];
PROC_ASSERT_HELD(p);
/*
* We are not going to read more than 2 * (PATH_MAX + ARG_MAX) bytes.
*/
nchr = 2 * (PATH_MAX + ARG_MAX);
error = get_proc_vector(td, p, &proc_vector, &vsize, type);
if (error != 0)
return (error);
for (done = 0, i = 0; i < (int)vsize && done < nchr; i++) {
/*
* The program may have scribbled into its argv array, e.g. to
* remove some arguments. If that has happened, break out
* before trying to read from NULL.
*/
if (proc_vector[i] == NULL)
break;
for (sptr = proc_vector[i]; ; sptr += GET_PS_STRINGS_CHUNK_SZ) {
error = proc_read_string(td, p, sptr, pss_string,
sizeof(pss_string));
if (error != 0)
goto done;
len = strnlen(pss_string, GET_PS_STRINGS_CHUNK_SZ);
if (done + len >= nchr)
len = nchr - done - 1;
sbuf_bcat(sb, pss_string, len);
if (len != GET_PS_STRINGS_CHUNK_SZ)
break;
done += GET_PS_STRINGS_CHUNK_SZ;
}
sbuf_bcat(sb, "", 1);
done += len + 1;
}
done:
free(proc_vector, M_TEMP);
return (error);
}
int
proc_getargv(struct thread *td, struct proc *p, struct sbuf *sb)
{
return (get_ps_strings(curthread, p, sb, PROC_ARG));
}
int
proc_getenvv(struct thread *td, struct proc *p, struct sbuf *sb)
{
return (get_ps_strings(curthread, p, sb, PROC_ENV));
}
int
proc_getauxv(struct thread *td, struct proc *p, struct sbuf *sb)
{
size_t vsize, size;
char **auxv;
int error;
error = get_proc_vector(td, p, &auxv, &vsize, PROC_AUX);
if (error == 0) {
#ifdef COMPAT_FREEBSD32
if (SV_PROC_FLAG(p, SV_ILP32) != 0)
size = vsize * sizeof(Elf32_Auxinfo);
else
#endif
size = vsize * sizeof(Elf_Auxinfo);
if (sbuf_bcat(sb, auxv, size) != 0)
error = ENOMEM;
free(auxv, M_TEMP);
}
return (error);
}
/*
* This sysctl allows a process to retrieve the argument list or process
* title for another process without groping around in the address space
* of the other process. It also allow a process to set its own "process
* title to a string of its own choice.
*/
static int
sysctl_kern_proc_args(SYSCTL_HANDLER_ARGS)
{
int *name = (int *)arg1;
u_int namelen = arg2;
struct pargs *newpa, *pa;
struct proc *p;
struct sbuf sb;
int flags, error = 0, error2;
pid_t pid;
if (namelen != 1)
return (EINVAL);
pid = (pid_t)name[0];
/*
* If the query is for this process and it is single-threaded, there
* is nobody to modify pargs, thus we can just read.
*/
p = curproc;
if (pid == p->p_pid && p->p_numthreads == 1 && req->newptr == NULL &&
(pa = p->p_args) != NULL)
return (SYSCTL_OUT(req, pa->ar_args, pa->ar_length));
flags = PGET_CANSEE;
if (req->newptr != NULL)
flags |= PGET_ISCURRENT;
error = pget(pid, flags, &p);
if (error)
return (error);
pa = p->p_args;
if (pa != NULL) {
pargs_hold(pa);
PROC_UNLOCK(p);
error = SYSCTL_OUT(req, pa->ar_args, pa->ar_length);
pargs_drop(pa);
} else if ((p->p_flag & (P_WEXIT | P_SYSTEM)) == 0) {
_PHOLD(p);
PROC_UNLOCK(p);
sbuf_new_for_sysctl(&sb, NULL, GET_PS_STRINGS_CHUNK_SZ, req);
sbuf_clear_flags(&sb, SBUF_INCLUDENUL);
error = proc_getargv(curthread, p, &sb);
error2 = sbuf_finish(&sb);
PRELE(p);
sbuf_delete(&sb);
if (error == 0 && error2 != 0)
error = error2;
} else {
PROC_UNLOCK(p);
}
if (error != 0 || req->newptr == NULL)
return (error);
if (req->newlen > ps_arg_cache_limit - sizeof(struct pargs))
return (ENOMEM);
if (req->newlen == 0) {
/*
* Clear the argument pointer, so that we'll fetch arguments
* with proc_getargv() until further notice.
*/
newpa = NULL;
} else {
newpa = pargs_alloc(req->newlen);
error = SYSCTL_IN(req, newpa->ar_args, req->newlen);
if (error != 0) {
pargs_free(newpa);
return (error);
}
}
PROC_LOCK(p);
pa = p->p_args;
p->p_args = newpa;
PROC_UNLOCK(p);
pargs_drop(pa);
return (0);
}
/*
* This sysctl allows a process to retrieve environment of another process.
*/
static int
sysctl_kern_proc_env(SYSCTL_HANDLER_ARGS)
{
int *name = (int *)arg1;
u_int namelen = arg2;
struct proc *p;
struct sbuf sb;
int error, error2;
if (namelen != 1)
return (EINVAL);
error = pget((pid_t)name[0], PGET_WANTREAD, &p);
if (error != 0)
return (error);
if ((p->p_flag & P_SYSTEM) != 0) {
PRELE(p);
return (0);
}
sbuf_new_for_sysctl(&sb, NULL, GET_PS_STRINGS_CHUNK_SZ, req);
sbuf_clear_flags(&sb, SBUF_INCLUDENUL);
error = proc_getenvv(curthread, p, &sb);
error2 = sbuf_finish(&sb);
PRELE(p);
sbuf_delete(&sb);
return (error != 0 ? error : error2);
}
/*
* This sysctl allows a process to retrieve ELF auxiliary vector of
* another process.
*/
static int
sysctl_kern_proc_auxv(SYSCTL_HANDLER_ARGS)
{
int *name = (int *)arg1;
u_int namelen = arg2;
struct proc *p;
struct sbuf sb;
int error, error2;
if (namelen != 1)
return (EINVAL);
error = pget((pid_t)name[0], PGET_WANTREAD, &p);
if (error != 0)
return (error);
if ((p->p_flag & P_SYSTEM) != 0) {
PRELE(p);
return (0);
}
sbuf_new_for_sysctl(&sb, NULL, GET_PS_STRINGS_CHUNK_SZ, req);
sbuf_clear_flags(&sb, SBUF_INCLUDENUL);
error = proc_getauxv(curthread, p, &sb);
error2 = sbuf_finish(&sb);
PRELE(p);
sbuf_delete(&sb);
return (error != 0 ? error : error2);
}
/*
* This sysctl allows a process to retrieve the path of the executable for
* itself or another process.
*/
static int
sysctl_kern_proc_pathname(SYSCTL_HANDLER_ARGS)
{
pid_t *pidp = (pid_t *)arg1;
unsigned int arglen = arg2;
struct proc *p;
struct vnode *vp;
char *retbuf, *freebuf;
int error;
if (arglen != 1)
return (EINVAL);
if (*pidp == -1) { /* -1 means this process */
p = req->td->td_proc;
} else {
error = pget(*pidp, PGET_CANSEE, &p);
if (error != 0)
return (error);
}
vp = p->p_textvp;
if (vp == NULL) {
if (*pidp != -1)
PROC_UNLOCK(p);
return (0);
}
vref(vp);
if (*pidp != -1)
PROC_UNLOCK(p);
error = vn_fullpath(req->td, vp, &retbuf, &freebuf);
vrele(vp);
if (error)
return (error);
error = SYSCTL_OUT(req, retbuf, strlen(retbuf) + 1);
free(freebuf, M_TEMP);
return (error);
}
static int
sysctl_kern_proc_sv_name(SYSCTL_HANDLER_ARGS)
{
struct proc *p;
char *sv_name;
int *name;
int namelen;
int error;
namelen = arg2;
if (namelen != 1)
return (EINVAL);
name = (int *)arg1;
error = pget((pid_t)name[0], PGET_CANSEE, &p);
if (error != 0)
return (error);
sv_name = p->p_sysent->sv_name;
PROC_UNLOCK(p);
return (sysctl_handle_string(oidp, sv_name, 0, req));
}
#ifdef KINFO_OVMENTRY_SIZE
CTASSERT(sizeof(struct kinfo_ovmentry) == KINFO_OVMENTRY_SIZE);
#endif
#ifdef COMPAT_FREEBSD7
static int
sysctl_kern_proc_ovmmap(SYSCTL_HANDLER_ARGS)
{
vm_map_entry_t entry, tmp_entry;
unsigned int last_timestamp;
char *fullpath, *freepath;
struct kinfo_ovmentry *kve;
struct vattr va;
struct ucred *cred;
int error, *name;
struct vnode *vp;
struct proc *p;
vm_map_t map;
struct vmspace *vm;
name = (int *)arg1;
error = pget((pid_t)name[0], PGET_WANTREAD, &p);
if (error != 0)
return (error);
vm = vmspace_acquire_ref(p);
if (vm == NULL) {
PRELE(p);
return (ESRCH);
}
kve = malloc(sizeof(*kve), M_TEMP, M_WAITOK);
map = &vm->vm_map;
vm_map_lock_read(map);
VM_MAP_ENTRY_FOREACH(entry, map) {
vm_object_t obj, tobj, lobj;
vm_offset_t addr;
if (entry->eflags & MAP_ENTRY_IS_SUB_MAP)
continue;
bzero(kve, sizeof(*kve));
kve->kve_structsize = sizeof(*kve);
kve->kve_private_resident = 0;
obj = entry->object.vm_object;
if (obj != NULL) {
VM_OBJECT_RLOCK(obj);
if (obj->shadow_count == 1)
kve->kve_private_resident =
obj->resident_page_count;
}
kve->kve_resident = 0;
addr = entry->start;
while (addr < entry->end) {
if (pmap_extract(map->pmap, addr))
kve->kve_resident++;
addr += PAGE_SIZE;
}
for (lobj = tobj = obj; tobj; tobj = tobj->backing_object) {
if (tobj != obj) {
VM_OBJECT_RLOCK(tobj);
kve->kve_offset += tobj->backing_object_offset;
}
if (lobj != obj)
VM_OBJECT_RUNLOCK(lobj);
lobj = tobj;
}
kve->kve_start = (void*)entry->start;
kve->kve_end = (void*)entry->end;
kve->kve_offset += (off_t)entry->offset;
if (entry->protection & VM_PROT_READ)
kve->kve_protection |= KVME_PROT_READ;
if (entry->protection & VM_PROT_WRITE)
kve->kve_protection |= KVME_PROT_WRITE;
if (entry->protection & VM_PROT_EXECUTE)
kve->kve_protection |= KVME_PROT_EXEC;
if (entry->eflags & MAP_ENTRY_COW)
kve->kve_flags |= KVME_FLAG_COW;
if (entry->eflags & MAP_ENTRY_NEEDS_COPY)
kve->kve_flags |= KVME_FLAG_NEEDS_COPY;
if (entry->eflags & MAP_ENTRY_NOCOREDUMP)
kve->kve_flags |= KVME_FLAG_NOCOREDUMP;
last_timestamp = map->timestamp;
vm_map_unlock_read(map);
kve->kve_fileid = 0;
kve->kve_fsid = 0;
freepath = NULL;
fullpath = "";
if (lobj) {
kve->kve_type = vm_object_kvme_type(lobj, &vp);
if (kve->kve_type == KVME_TYPE_MGTDEVICE)
kve->kve_type = KVME_TYPE_UNKNOWN;
if (vp != NULL)
vref(vp);
if (lobj != obj)
VM_OBJECT_RUNLOCK(lobj);
kve->kve_ref_count = obj->ref_count;
kve->kve_shadow_count = obj->shadow_count;
VM_OBJECT_RUNLOCK(obj);
if (vp != NULL) {
vn_fullpath(curthread, vp, &fullpath,
&freepath);
cred = curthread->td_ucred;
vn_lock(vp, LK_SHARED | LK_RETRY);
if (VOP_GETATTR(vp, &va, cred) == 0) {
kve->kve_fileid = va.va_fileid;
/* truncate */
kve->kve_fsid = va.va_fsid;
}
vput(vp);
}
} else {
kve->kve_type = KVME_TYPE_NONE;
kve->kve_ref_count = 0;
kve->kve_shadow_count = 0;
}
strlcpy(kve->kve_path, fullpath, sizeof(kve->kve_path));
if (freepath != NULL)
free(freepath, M_TEMP);
error = SYSCTL_OUT(req, kve, sizeof(*kve));
vm_map_lock_read(map);
if (error)
break;
if (last_timestamp != map->timestamp) {
vm_map_lookup_entry(map, addr - 1, &tmp_entry);
entry = tmp_entry;
}
}
vm_map_unlock_read(map);
vmspace_free(vm);
PRELE(p);
free(kve, M_TEMP);
return (error);
}
#endif /* COMPAT_FREEBSD7 */
#ifdef KINFO_VMENTRY_SIZE
CTASSERT(sizeof(struct kinfo_vmentry) == KINFO_VMENTRY_SIZE);
#endif
void
kern_proc_vmmap_resident(vm_map_t map, vm_map_entry_t entry,
int *resident_count, bool *super)
{
vm_object_t obj, tobj;
vm_page_t m, m_adv;
vm_offset_t addr;
vm_paddr_t pa;
vm_pindex_t pi, pi_adv, pindex;
*super = false;
*resident_count = 0;
if (vmmap_skip_res_cnt)
return;
pa = 0;
obj = entry->object.vm_object;
addr = entry->start;
m_adv = NULL;
pi = OFF_TO_IDX(entry->offset);
for (; addr < entry->end; addr += IDX_TO_OFF(pi_adv), pi += pi_adv) {
if (m_adv != NULL) {
m = m_adv;
} else {
pi_adv = atop(entry->end - addr);
pindex = pi;
for (tobj = obj;; tobj = tobj->backing_object) {
m = vm_page_find_least(tobj, pindex);
if (m != NULL) {
if (m->pindex == pindex)
break;
if (pi_adv > m->pindex - pindex) {
pi_adv = m->pindex - pindex;
m_adv = m;
}
}
if (tobj->backing_object == NULL)
goto next;
pindex += OFF_TO_IDX(tobj->
backing_object_offset);
}
}
m_adv = NULL;
if (m->psind != 0 && addr + pagesizes[1] <= entry->end &&
(addr & (pagesizes[1] - 1)) == 0 &&
(pmap_mincore(map->pmap, addr, &pa) & MINCORE_SUPER) != 0) {
*super = true;
pi_adv = atop(pagesizes[1]);
} else {
/*
* We do not test the found page on validity.
* Either the page is busy and being paged in,
* or it was invalidated. The first case
* should be counted as resident, the second
* is not so clear; we do account both.
*/
pi_adv = 1;
}
*resident_count += pi_adv;
next:;
}
}
/*
* Must be called with the process locked and will return unlocked.
*/
int
kern_proc_vmmap_out(struct proc *p, struct sbuf *sb, ssize_t maxlen, int flags)
{
vm_map_entry_t entry, tmp_entry;
struct vattr va;
vm_map_t map;
vm_object_t obj, tobj, lobj;
char *fullpath, *freepath;
struct kinfo_vmentry *kve;
struct ucred *cred;
struct vnode *vp;
struct vmspace *vm;
vm_offset_t addr;
unsigned int last_timestamp;
int error;
bool super;
PROC_LOCK_ASSERT(p, MA_OWNED);
_PHOLD(p);
PROC_UNLOCK(p);
vm = vmspace_acquire_ref(p);
if (vm == NULL) {
PRELE(p);
return (ESRCH);
}
kve = malloc(sizeof(*kve), M_TEMP, M_WAITOK | M_ZERO);
error = 0;
map = &vm->vm_map;
vm_map_lock_read(map);
VM_MAP_ENTRY_FOREACH(entry, map) {
if (entry->eflags & MAP_ENTRY_IS_SUB_MAP)
continue;
addr = entry->end;
bzero(kve, sizeof(*kve));
obj = entry->object.vm_object;
if (obj != NULL) {
for (tobj = obj; tobj != NULL;
tobj = tobj->backing_object) {
VM_OBJECT_RLOCK(tobj);
kve->kve_offset += tobj->backing_object_offset;
lobj = tobj;
}
if (obj->backing_object == NULL)
kve->kve_private_resident =
obj->resident_page_count;
kern_proc_vmmap_resident(map, entry,
&kve->kve_resident, &super);
if (super)
kve->kve_flags |= KVME_FLAG_SUPER;
for (tobj = obj; tobj != NULL;
tobj = tobj->backing_object) {
if (tobj != obj && tobj != lobj)
VM_OBJECT_RUNLOCK(tobj);
}
} else {
lobj = NULL;
}
kve->kve_start = entry->start;
kve->kve_end = entry->end;
kve->kve_offset += entry->offset;
if (entry->protection & VM_PROT_READ)
kve->kve_protection |= KVME_PROT_READ;
if (entry->protection & VM_PROT_WRITE)
kve->kve_protection |= KVME_PROT_WRITE;
if (entry->protection & VM_PROT_EXECUTE)
kve->kve_protection |= KVME_PROT_EXEC;
if (entry->eflags & MAP_ENTRY_COW)
kve->kve_flags |= KVME_FLAG_COW;
if (entry->eflags & MAP_ENTRY_NEEDS_COPY)
kve->kve_flags |= KVME_FLAG_NEEDS_COPY;
if (entry->eflags & MAP_ENTRY_NOCOREDUMP)
kve->kve_flags |= KVME_FLAG_NOCOREDUMP;
if (entry->eflags & MAP_ENTRY_GROWS_UP)
kve->kve_flags |= KVME_FLAG_GROWS_UP;
if (entry->eflags & MAP_ENTRY_GROWS_DOWN)
kve->kve_flags |= KVME_FLAG_GROWS_DOWN;
if (entry->eflags & MAP_ENTRY_USER_WIRED)
kve->kve_flags |= KVME_FLAG_USER_WIRED;
last_timestamp = map->timestamp;
vm_map_unlock_read(map);
freepath = NULL;
fullpath = "";
if (lobj != NULL) {
kve->kve_type = vm_object_kvme_type(lobj, &vp);
if (vp != NULL)
vref(vp);
if (lobj != obj)
VM_OBJECT_RUNLOCK(lobj);
kve->kve_ref_count = obj->ref_count;
kve->kve_shadow_count = obj->shadow_count;
VM_OBJECT_RUNLOCK(obj);
if (vp != NULL) {
vn_fullpath(curthread, vp, &fullpath,
&freepath);
kve->kve_vn_type = vntype_to_kinfo(vp->v_type);
cred = curthread->td_ucred;
vn_lock(vp, LK_SHARED | LK_RETRY);
if (VOP_GETATTR(vp, &va, cred) == 0) {
kve->kve_vn_fileid = va.va_fileid;
kve->kve_vn_fsid = va.va_fsid;
kve->kve_vn_fsid_freebsd11 =
kve->kve_vn_fsid; /* truncate */
kve->kve_vn_mode =
MAKEIMODE(va.va_type, va.va_mode);
kve->kve_vn_size = va.va_size;
kve->kve_vn_rdev = va.va_rdev;
kve->kve_vn_rdev_freebsd11 =
kve->kve_vn_rdev; /* truncate */
kve->kve_status = KF_ATTR_VALID;
}
vput(vp);
}
} else {
kve->kve_type = KVME_TYPE_NONE;
kve->kve_ref_count = 0;
kve->kve_shadow_count = 0;
}
strlcpy(kve->kve_path, fullpath, sizeof(kve->kve_path));
if (freepath != NULL)
free(freepath, M_TEMP);
/* Pack record size down */
if ((flags & KERN_VMMAP_PACK_KINFO) != 0)
kve->kve_structsize =
offsetof(struct kinfo_vmentry, kve_path) +
strlen(kve->kve_path) + 1;
else
kve->kve_structsize = sizeof(*kve);
kve->kve_structsize = roundup(kve->kve_structsize,
sizeof(uint64_t));
/* Halt filling and truncate rather than exceeding maxlen */
if (maxlen != -1 && maxlen < kve->kve_structsize) {
error = 0;
vm_map_lock_read(map);
break;
} else if (maxlen != -1)
maxlen -= kve->kve_structsize;
if (sbuf_bcat(sb, kve, kve->kve_structsize) != 0)
error = ENOMEM;
vm_map_lock_read(map);
if (error != 0)
break;
if (last_timestamp != map->timestamp) {
vm_map_lookup_entry(map, addr - 1, &tmp_entry);
entry = tmp_entry;
}
}
vm_map_unlock_read(map);
vmspace_free(vm);
PRELE(p);
free(kve, M_TEMP);
return (error);
}
static int
sysctl_kern_proc_vmmap(SYSCTL_HANDLER_ARGS)
{
struct proc *p;
struct sbuf sb;
int error, error2, *name;
name = (int *)arg1;
sbuf_new_for_sysctl(&sb, NULL, sizeof(struct kinfo_vmentry), req);
sbuf_clear_flags(&sb, SBUF_INCLUDENUL);
error = pget((pid_t)name[0], PGET_CANDEBUG | PGET_NOTWEXIT, &p);
if (error != 0) {
sbuf_delete(&sb);
return (error);
}
error = kern_proc_vmmap_out(p, &sb, -1, KERN_VMMAP_PACK_KINFO);
error2 = sbuf_finish(&sb);
sbuf_delete(&sb);
return (error != 0 ? error : error2);
}
#if defined(STACK) || defined(DDB)
static int
sysctl_kern_proc_kstack(SYSCTL_HANDLER_ARGS)
{
struct kinfo_kstack *kkstp;
int error, i, *name, numthreads;
lwpid_t *lwpidarray;
struct thread *td;
struct stack *st;
struct sbuf sb;
struct proc *p;
name = (int *)arg1;
error = pget((pid_t)name[0], PGET_NOTINEXEC | PGET_WANTREAD, &p);
if (error != 0)
return (error);
kkstp = malloc(sizeof(*kkstp), M_TEMP, M_WAITOK);
st = stack_create(M_WAITOK);
lwpidarray = NULL;
PROC_LOCK(p);
do {
if (lwpidarray != NULL) {
free(lwpidarray, M_TEMP);
lwpidarray = NULL;
}
numthreads = p->p_numthreads;
PROC_UNLOCK(p);
lwpidarray = malloc(sizeof(*lwpidarray) * numthreads, M_TEMP,
M_WAITOK | M_ZERO);
PROC_LOCK(p);
} while (numthreads < p->p_numthreads);
/*
* XXXRW: During the below loop, execve(2) and countless other sorts
* of changes could have taken place. Should we check to see if the
* vmspace has been replaced, or the like, in order to prevent
* giving a snapshot that spans, say, execve(2), with some threads
* before and some after? Among other things, the credentials could
* have changed, in which case the right to extract debug info might
* no longer be assured.
*/
i = 0;
FOREACH_THREAD_IN_PROC(p, td) {
KASSERT(i < numthreads,
("sysctl_kern_proc_kstack: numthreads"));
lwpidarray[i] = td->td_tid;
i++;
}
numthreads = i;
for (i = 0; i < numthreads; i++) {
td = thread_find(p, lwpidarray[i]);
if (td == NULL) {
continue;
}
bzero(kkstp, sizeof(*kkstp));
(void)sbuf_new(&sb, kkstp->kkst_trace,
sizeof(kkstp->kkst_trace), SBUF_FIXEDLEN);
thread_lock(td);
kkstp->kkst_tid = td->td_tid;
if (TD_IS_SWAPPED(td)) {
kkstp->kkst_state = KKST_STATE_SWAPPED;
} else if (TD_IS_RUNNING(td)) {
if (stack_save_td_running(st, td) == 0)
kkstp->kkst_state = KKST_STATE_STACKOK;
else
kkstp->kkst_state = KKST_STATE_RUNNING;
} else {
kkstp->kkst_state = KKST_STATE_STACKOK;
stack_save_td(st, td);
}
thread_unlock(td);
PROC_UNLOCK(p);
stack_sbuf_print(&sb, st);
sbuf_finish(&sb);
sbuf_delete(&sb);
error = SYSCTL_OUT(req, kkstp, sizeof(*kkstp));
PROC_LOCK(p);
if (error)
break;
}
_PRELE(p);
PROC_UNLOCK(p);
if (lwpidarray != NULL)
free(lwpidarray, M_TEMP);
stack_destroy(st);
free(kkstp, M_TEMP);
return (error);
}
#endif
/*
* This sysctl allows a process to retrieve the full list of groups from
* itself or another process.
*/
static int
sysctl_kern_proc_groups(SYSCTL_HANDLER_ARGS)
{
pid_t *pidp = (pid_t *)arg1;
unsigned int arglen = arg2;
struct proc *p;
struct ucred *cred;
int error;
if (arglen != 1)
return (EINVAL);
if (*pidp == -1) { /* -1 means this process */
p = req->td->td_proc;
PROC_LOCK(p);
} else {
error = pget(*pidp, PGET_CANSEE, &p);
if (error != 0)
return (error);
}
cred = crhold(p->p_ucred);
PROC_UNLOCK(p);
error = SYSCTL_OUT(req, cred->cr_groups,
cred->cr_ngroups * sizeof(gid_t));
crfree(cred);
return (error);
}
/*
* This sysctl allows a process to retrieve or/and set the resource limit for
* another process.
*/
static int
sysctl_kern_proc_rlimit(SYSCTL_HANDLER_ARGS)
{
int *name = (int *)arg1;
u_int namelen = arg2;
struct rlimit rlim;
struct proc *p;
u_int which;
int flags, error;
if (namelen != 2)
return (EINVAL);
which = (u_int)name[1];
if (which >= RLIM_NLIMITS)
return (EINVAL);
if (req->newptr != NULL && req->newlen != sizeof(rlim))
return (EINVAL);
flags = PGET_HOLD | PGET_NOTWEXIT;
if (req->newptr != NULL)
flags |= PGET_CANDEBUG;
else
flags |= PGET_CANSEE;
error = pget((pid_t)name[0], flags, &p);
if (error != 0)
return (error);
/*
* Retrieve limit.
*/
if (req->oldptr != NULL) {
PROC_LOCK(p);
lim_rlimit_proc(p, which, &rlim);
PROC_UNLOCK(p);
}
error = SYSCTL_OUT(req, &rlim, sizeof(rlim));
if (error != 0)
goto errout;
/*
* Set limit.
*/
if (req->newptr != NULL) {
error = SYSCTL_IN(req, &rlim, sizeof(rlim));
if (error == 0)
error = kern_proc_setrlimit(curthread, p, which, &rlim);
}
errout:
PRELE(p);
return (error);
}
/*
* This sysctl allows a process to retrieve ps_strings structure location of
* another process.
*/
static int
sysctl_kern_proc_ps_strings(SYSCTL_HANDLER_ARGS)
{
int *name = (int *)arg1;
u_int namelen = arg2;
struct proc *p;
vm_offset_t ps_strings;
int error;
#ifdef COMPAT_FREEBSD32
uint32_t ps_strings32;
#endif
if (namelen != 1)
return (EINVAL);
error = pget((pid_t)name[0], PGET_CANDEBUG, &p);
if (error != 0)
return (error);
#ifdef COMPAT_FREEBSD32
if ((req->flags & SCTL_MASK32) != 0) {
/*
* We return 0 if the 32 bit emulation request is for a 64 bit
* process.
*/
ps_strings32 = SV_PROC_FLAG(p, SV_ILP32) != 0 ?
PTROUT(p->p_sysent->sv_psstrings) : 0;
PROC_UNLOCK(p);
error = SYSCTL_OUT(req, &ps_strings32, sizeof(ps_strings32));
return (error);
}
#endif
ps_strings = p->p_sysent->sv_psstrings;
PROC_UNLOCK(p);
error = SYSCTL_OUT(req, &ps_strings, sizeof(ps_strings));
return (error);
}
/*
* This sysctl allows a process to retrieve umask of another process.
*/
static int
sysctl_kern_proc_umask(SYSCTL_HANDLER_ARGS)
{
int *name = (int *)arg1;
u_int namelen = arg2;
struct proc *p;
int error;
u_short fd_cmask;
pid_t pid;
if (namelen != 1)
return (EINVAL);
pid = (pid_t)name[0];
p = curproc;
if (pid == p->p_pid || pid == 0) {
fd_cmask = p->p_fd->fd_cmask;
goto out;
}
error = pget(pid, PGET_WANTREAD, &p);
if (error != 0)
return (error);
fd_cmask = p->p_fd->fd_cmask;
PRELE(p);
out:
error = SYSCTL_OUT(req, &fd_cmask, sizeof(fd_cmask));
return (error);
}
/*
* This sysctl allows a process to set and retrieve binary osreldate of
* another process.
*/
static int
sysctl_kern_proc_osrel(SYSCTL_HANDLER_ARGS)
{
int *name = (int *)arg1;
u_int namelen = arg2;
struct proc *p;
int flags, error, osrel;
if (namelen != 1)
return (EINVAL);
if (req->newptr != NULL && req->newlen != sizeof(osrel))
return (EINVAL);
flags = PGET_HOLD | PGET_NOTWEXIT;
if (req->newptr != NULL)
flags |= PGET_CANDEBUG;
else
flags |= PGET_CANSEE;
error = pget((pid_t)name[0], flags, &p);
if (error != 0)
return (error);
error = SYSCTL_OUT(req, &p->p_osrel, sizeof(p->p_osrel));
if (error != 0)
goto errout;
if (req->newptr != NULL) {
error = SYSCTL_IN(req, &osrel, sizeof(osrel));
if (error != 0)
goto errout;
if (osrel < 0) {
error = EINVAL;
goto errout;
}
p->p_osrel = osrel;
}
errout:
PRELE(p);
return (error);
}
static int
sysctl_kern_proc_sigtramp(SYSCTL_HANDLER_ARGS)
{
int *name = (int *)arg1;
u_int namelen = arg2;
struct proc *p;
struct kinfo_sigtramp kst;
const struct sysentvec *sv;
int error;
#ifdef COMPAT_FREEBSD32
struct kinfo_sigtramp32 kst32;
#endif
if (namelen != 1)
return (EINVAL);
error = pget((pid_t)name[0], PGET_CANDEBUG, &p);
if (error != 0)
return (error);
sv = p->p_sysent;
#ifdef COMPAT_FREEBSD32
if ((req->flags & SCTL_MASK32) != 0) {
bzero(&kst32, sizeof(kst32));
if (SV_PROC_FLAG(p, SV_ILP32)) {
if (sv->sv_sigcode_base != 0) {
kst32.ksigtramp_start = sv->sv_sigcode_base;
kst32.ksigtramp_end = sv->sv_sigcode_base +
*sv->sv_szsigcode;
} else {
kst32.ksigtramp_start = sv->sv_psstrings -
*sv->sv_szsigcode;
kst32.ksigtramp_end = sv->sv_psstrings;
}
}
PROC_UNLOCK(p);
error = SYSCTL_OUT(req, &kst32, sizeof(kst32));
return (error);
}
#endif
bzero(&kst, sizeof(kst));
if (sv->sv_sigcode_base != 0) {
kst.ksigtramp_start = (char *)sv->sv_sigcode_base;
kst.ksigtramp_end = (char *)sv->sv_sigcode_base +
*sv->sv_szsigcode;
} else {
kst.ksigtramp_start = (char *)sv->sv_psstrings -
*sv->sv_szsigcode;
kst.ksigtramp_end = (char *)sv->sv_psstrings;
}
PROC_UNLOCK(p);
error = SYSCTL_OUT(req, &kst, sizeof(kst));
return (error);
}
SYSCTL_NODE(_kern, KERN_PROC, proc, CTLFLAG_RD, 0, "Process table");
SYSCTL_PROC(_kern_proc, KERN_PROC_ALL, all, CTLFLAG_RD|CTLTYPE_STRUCT|
CTLFLAG_MPSAFE, 0, 0, sysctl_kern_proc, "S,proc",
"Return entire process table");
static SYSCTL_NODE(_kern_proc, KERN_PROC_GID, gid, CTLFLAG_RD | CTLFLAG_MPSAFE,
sysctl_kern_proc, "Process table");
static SYSCTL_NODE(_kern_proc, KERN_PROC_PGRP, pgrp, CTLFLAG_RD | CTLFLAG_MPSAFE,
sysctl_kern_proc, "Process table");
static SYSCTL_NODE(_kern_proc, KERN_PROC_RGID, rgid, CTLFLAG_RD | CTLFLAG_MPSAFE,
sysctl_kern_proc, "Process table");
static SYSCTL_NODE(_kern_proc, KERN_PROC_SESSION, sid, CTLFLAG_RD |
CTLFLAG_MPSAFE, sysctl_kern_proc, "Process table");
static SYSCTL_NODE(_kern_proc, KERN_PROC_TTY, tty, CTLFLAG_RD | CTLFLAG_MPSAFE,
sysctl_kern_proc, "Process table");
static SYSCTL_NODE(_kern_proc, KERN_PROC_UID, uid, CTLFLAG_RD | CTLFLAG_MPSAFE,
sysctl_kern_proc, "Process table");
static SYSCTL_NODE(_kern_proc, KERN_PROC_RUID, ruid, CTLFLAG_RD | CTLFLAG_MPSAFE,
sysctl_kern_proc, "Process table");
static SYSCTL_NODE(_kern_proc, KERN_PROC_PID, pid, CTLFLAG_RD | CTLFLAG_MPSAFE,
sysctl_kern_proc, "Process table");
static SYSCTL_NODE(_kern_proc, KERN_PROC_PROC, proc, CTLFLAG_RD | CTLFLAG_MPSAFE,
sysctl_kern_proc, "Return process table, no threads");
static SYSCTL_NODE(_kern_proc, KERN_PROC_ARGS, args,
CTLFLAG_RW | CTLFLAG_CAPWR | CTLFLAG_ANYBODY | CTLFLAG_MPSAFE,
sysctl_kern_proc_args, "Process argument list");
static SYSCTL_NODE(_kern_proc, KERN_PROC_ENV, env, CTLFLAG_RD | CTLFLAG_MPSAFE,
sysctl_kern_proc_env, "Process environment");
static SYSCTL_NODE(_kern_proc, KERN_PROC_AUXV, auxv, CTLFLAG_RD |
CTLFLAG_MPSAFE, sysctl_kern_proc_auxv, "Process ELF auxiliary vector");
static SYSCTL_NODE(_kern_proc, KERN_PROC_PATHNAME, pathname, CTLFLAG_RD |
CTLFLAG_MPSAFE, sysctl_kern_proc_pathname, "Process executable path");
static SYSCTL_NODE(_kern_proc, KERN_PROC_SV_NAME, sv_name, CTLFLAG_RD |
CTLFLAG_MPSAFE, sysctl_kern_proc_sv_name,
"Process syscall vector name (ABI type)");
static SYSCTL_NODE(_kern_proc, (KERN_PROC_GID | KERN_PROC_INC_THREAD), gid_td,
CTLFLAG_RD | CTLFLAG_MPSAFE, sysctl_kern_proc, "Process table");
static SYSCTL_NODE(_kern_proc, (KERN_PROC_PGRP | KERN_PROC_INC_THREAD), pgrp_td,
CTLFLAG_RD | CTLFLAG_MPSAFE, sysctl_kern_proc, "Process table");
static SYSCTL_NODE(_kern_proc, (KERN_PROC_RGID | KERN_PROC_INC_THREAD), rgid_td,
CTLFLAG_RD | CTLFLAG_MPSAFE, sysctl_kern_proc, "Process table");
static SYSCTL_NODE(_kern_proc, (KERN_PROC_SESSION | KERN_PROC_INC_THREAD),
sid_td, CTLFLAG_RD | CTLFLAG_MPSAFE, sysctl_kern_proc, "Process table");
static SYSCTL_NODE(_kern_proc, (KERN_PROC_TTY | KERN_PROC_INC_THREAD), tty_td,
CTLFLAG_RD | CTLFLAG_MPSAFE, sysctl_kern_proc, "Process table");
static SYSCTL_NODE(_kern_proc, (KERN_PROC_UID | KERN_PROC_INC_THREAD), uid_td,
CTLFLAG_RD | CTLFLAG_MPSAFE, sysctl_kern_proc, "Process table");
static SYSCTL_NODE(_kern_proc, (KERN_PROC_RUID | KERN_PROC_INC_THREAD), ruid_td,
CTLFLAG_RD | CTLFLAG_MPSAFE, sysctl_kern_proc, "Process table");
static SYSCTL_NODE(_kern_proc, (KERN_PROC_PID | KERN_PROC_INC_THREAD), pid_td,
CTLFLAG_RD | CTLFLAG_MPSAFE, sysctl_kern_proc, "Process table");
static SYSCTL_NODE(_kern_proc, (KERN_PROC_PROC | KERN_PROC_INC_THREAD), proc_td,
CTLFLAG_RD | CTLFLAG_MPSAFE, sysctl_kern_proc,
"Return process table, no threads");
#ifdef COMPAT_FREEBSD7
static SYSCTL_NODE(_kern_proc, KERN_PROC_OVMMAP, ovmmap, CTLFLAG_RD |
CTLFLAG_MPSAFE, sysctl_kern_proc_ovmmap, "Old Process vm map entries");
#endif
static SYSCTL_NODE(_kern_proc, KERN_PROC_VMMAP, vmmap, CTLFLAG_RD |
CTLFLAG_MPSAFE, sysctl_kern_proc_vmmap, "Process vm map entries");
#if defined(STACK) || defined(DDB)
static SYSCTL_NODE(_kern_proc, KERN_PROC_KSTACK, kstack, CTLFLAG_RD |
CTLFLAG_MPSAFE, sysctl_kern_proc_kstack, "Process kernel stacks");
#endif
static SYSCTL_NODE(_kern_proc, KERN_PROC_GROUPS, groups, CTLFLAG_RD |
CTLFLAG_MPSAFE, sysctl_kern_proc_groups, "Process groups");
static SYSCTL_NODE(_kern_proc, KERN_PROC_RLIMIT, rlimit, CTLFLAG_RW |
CTLFLAG_ANYBODY | CTLFLAG_MPSAFE, sysctl_kern_proc_rlimit,
"Process resource limits");
static SYSCTL_NODE(_kern_proc, KERN_PROC_PS_STRINGS, ps_strings, CTLFLAG_RD |
CTLFLAG_MPSAFE, sysctl_kern_proc_ps_strings,
"Process ps_strings location");
static SYSCTL_NODE(_kern_proc, KERN_PROC_UMASK, umask, CTLFLAG_RD |
CTLFLAG_MPSAFE, sysctl_kern_proc_umask, "Process umask");
static SYSCTL_NODE(_kern_proc, KERN_PROC_OSREL, osrel, CTLFLAG_RW |
CTLFLAG_ANYBODY | CTLFLAG_MPSAFE, sysctl_kern_proc_osrel,
"Process binary osreldate");
static SYSCTL_NODE(_kern_proc, KERN_PROC_SIGTRAMP, sigtramp, CTLFLAG_RD |
CTLFLAG_MPSAFE, sysctl_kern_proc_sigtramp,
"Process signal trampoline location");
int allproc_gen;
/*
* stop_all_proc() purpose is to stop all process which have usermode,
* except current process for obvious reasons. This makes it somewhat
* unreliable when invoked from multithreaded process. The service
* must not be user-callable anyway.
*/
void
stop_all_proc(void)
{
struct proc *cp, *p;
int r, gen;
bool restart, seen_stopped, seen_exiting, stopped_some;
cp = curproc;
allproc_loop:
sx_xlock(&allproc_lock);
gen = allproc_gen;
seen_exiting = seen_stopped = stopped_some = restart = false;
LIST_REMOVE(cp, p_list);
LIST_INSERT_HEAD(&allproc, cp, p_list);
for (;;) {
p = LIST_NEXT(cp, p_list);
if (p == NULL)
break;
LIST_REMOVE(cp, p_list);
LIST_INSERT_AFTER(p, cp, p_list);
PROC_LOCK(p);
if ((p->p_flag & (P_KPROC | P_SYSTEM | P_TOTAL_STOP)) != 0) {
PROC_UNLOCK(p);
continue;
}
if ((p->p_flag & P_WEXIT) != 0) {
seen_exiting = true;
PROC_UNLOCK(p);
continue;
}
if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
/*
* Stopped processes are tolerated when there
* are no other processes which might continue
* them. P_STOPPED_SINGLE but not
* P_TOTAL_STOP process still has at least one
* thread running.
*/
seen_stopped = true;
PROC_UNLOCK(p);
continue;
}
sx_xunlock(&allproc_lock);
_PHOLD(p);
r = thread_single(p, SINGLE_ALLPROC);
if (r != 0)
restart = true;
else
stopped_some = true;
_PRELE(p);
PROC_UNLOCK(p);
sx_xlock(&allproc_lock);
}
/* Catch forked children we did not see in iteration. */
if (gen != allproc_gen)
restart = true;
sx_xunlock(&allproc_lock);
if (restart || stopped_some || seen_exiting || seen_stopped) {
kern_yield(PRI_USER);
goto allproc_loop;
}
}
void
resume_all_proc(void)
{
struct proc *cp, *p;
cp = curproc;
sx_xlock(&allproc_lock);
again:
LIST_REMOVE(cp, p_list);
LIST_INSERT_HEAD(&allproc, cp, p_list);
for (;;) {
p = LIST_NEXT(cp, p_list);
if (p == NULL)
break;
LIST_REMOVE(cp, p_list);
LIST_INSERT_AFTER(p, cp, p_list);
PROC_LOCK(p);
if ((p->p_flag & P_TOTAL_STOP) != 0) {
sx_xunlock(&allproc_lock);
_PHOLD(p);
thread_single_end(p, SINGLE_ALLPROC);
_PRELE(p);
PROC_UNLOCK(p);
sx_xlock(&allproc_lock);
} else {
PROC_UNLOCK(p);
}
}
/* Did the loop above missed any stopped process ? */
FOREACH_PROC_IN_SYSTEM(p) {
/* No need for proc lock. */
if ((p->p_flag & P_TOTAL_STOP) != 0)
goto again;
}
sx_xunlock(&allproc_lock);
}
/* #define TOTAL_STOP_DEBUG 1 */
#ifdef TOTAL_STOP_DEBUG
volatile static int ap_resume;
#include <sys/mount.h>
static int
sysctl_debug_stop_all_proc(SYSCTL_HANDLER_ARGS)
{
int error, val;
val = 0;
ap_resume = 0;
error = sysctl_handle_int(oidp, &val, 0, req);
if (error != 0 || req->newptr == NULL)
return (error);
if (val != 0) {
stop_all_proc();
syncer_suspend();
while (ap_resume == 0)
;
syncer_resume();
resume_all_proc();
}
return (0);
}
SYSCTL_PROC(_debug, OID_AUTO, stop_all_proc, CTLTYPE_INT | CTLFLAG_RW |
CTLFLAG_MPSAFE, __DEVOLATILE(int *, &ap_resume), 0,
sysctl_debug_stop_all_proc, "I",
"");
#endif
Index: head/sys/kern/kern_sx.c
===================================================================
--- head/sys/kern/kern_sx.c (revision 356056)
+++ head/sys/kern/kern_sx.c (revision 356057)
@@ -1,1551 +1,1551 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2007 Attilio Rao <attilio@freebsd.org>
* Copyright (c) 2001 Jason Evans <jasone@freebsd.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice(s), this list of conditions and the following disclaimer as
* the first lines of this file unmodified other than the possible
* addition of one or more copyright notices.
* 2. Redistributions in binary form must reproduce the above copyright
* notice(s), this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
* DAMAGE.
*/
/*
* Shared/exclusive locks. This implementation attempts to ensure
* deterministic lock granting behavior, so that slocks and xlocks are
* interleaved.
*
* Priority propagation will not generally raise the priority of lock holders,
* so should not be relied upon in combination with sx locks.
*/
#include "opt_ddb.h"
#include "opt_hwpmc_hooks.h"
#include "opt_no_adaptive_sx.h"
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/sched.h>
#include <sys/sleepqueue.h>
#include <sys/sx.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#if defined(SMP) && !defined(NO_ADAPTIVE_SX)
#include <machine/cpu.h>
#endif
#ifdef DDB
#include <ddb/ddb.h>
#endif
#if defined(SMP) && !defined(NO_ADAPTIVE_SX)
#define ADAPTIVE_SX
#endif
#ifdef HWPMC_HOOKS
#include <sys/pmckern.h>
PMC_SOFT_DECLARE( , , lock, failed);
#endif
/* Handy macros for sleep queues. */
#define SQ_EXCLUSIVE_QUEUE 0
#define SQ_SHARED_QUEUE 1
/*
* Variations on DROP_GIANT()/PICKUP_GIANT() for use in this file. We
* drop Giant anytime we have to sleep or if we adaptively spin.
*/
#define GIANT_DECLARE \
int _giantcnt = 0; \
WITNESS_SAVE_DECL(Giant) \
#define GIANT_SAVE(work) do { \
if (__predict_false(mtx_owned(&Giant))) { \
work++; \
WITNESS_SAVE(&Giant.lock_object, Giant); \
while (mtx_owned(&Giant)) { \
_giantcnt++; \
mtx_unlock(&Giant); \
} \
} \
} while (0)
#define GIANT_RESTORE() do { \
if (_giantcnt > 0) { \
mtx_assert(&Giant, MA_NOTOWNED); \
while (_giantcnt--) \
mtx_lock(&Giant); \
WITNESS_RESTORE(&Giant.lock_object, Giant); \
} \
} while (0)
/*
* Returns true if an exclusive lock is recursed. It assumes
* curthread currently has an exclusive lock.
*/
#define sx_recursed(sx) ((sx)->sx_recurse != 0)
static void assert_sx(const struct lock_object *lock, int what);
#ifdef DDB
static void db_show_sx(const struct lock_object *lock);
#endif
static void lock_sx(struct lock_object *lock, uintptr_t how);
#ifdef KDTRACE_HOOKS
static int owner_sx(const struct lock_object *lock, struct thread **owner);
#endif
static uintptr_t unlock_sx(struct lock_object *lock);
struct lock_class lock_class_sx = {
.lc_name = "sx",
.lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE | LC_UPGRADABLE,
.lc_assert = assert_sx,
#ifdef DDB
.lc_ddb_show = db_show_sx,
#endif
.lc_lock = lock_sx,
.lc_unlock = unlock_sx,
#ifdef KDTRACE_HOOKS
.lc_owner = owner_sx,
#endif
};
#ifndef INVARIANTS
#define _sx_assert(sx, what, file, line)
#endif
#ifdef ADAPTIVE_SX
static __read_frequently u_int asx_retries;
static __read_frequently u_int asx_loops;
static SYSCTL_NODE(_debug, OID_AUTO, sx, CTLFLAG_RD, NULL, "sxlock debugging");
SYSCTL_UINT(_debug_sx, OID_AUTO, retries, CTLFLAG_RW, &asx_retries, 0, "");
SYSCTL_UINT(_debug_sx, OID_AUTO, loops, CTLFLAG_RW, &asx_loops, 0, "");
static struct lock_delay_config __read_frequently sx_delay;
SYSCTL_INT(_debug_sx, OID_AUTO, delay_base, CTLFLAG_RW, &sx_delay.base,
0, "");
SYSCTL_INT(_debug_sx, OID_AUTO, delay_max, CTLFLAG_RW, &sx_delay.max,
0, "");
static void
sx_lock_delay_init(void *arg __unused)
{
lock_delay_default_init(&sx_delay);
asx_retries = 10;
asx_loops = max(10000, sx_delay.max);
}
LOCK_DELAY_SYSINIT(sx_lock_delay_init);
#endif
void
assert_sx(const struct lock_object *lock, int what)
{
sx_assert((const struct sx *)lock, what);
}
void
lock_sx(struct lock_object *lock, uintptr_t how)
{
struct sx *sx;
sx = (struct sx *)lock;
if (how)
sx_slock(sx);
else
sx_xlock(sx);
}
uintptr_t
unlock_sx(struct lock_object *lock)
{
struct sx *sx;
sx = (struct sx *)lock;
sx_assert(sx, SA_LOCKED | SA_NOTRECURSED);
if (sx_xlocked(sx)) {
sx_xunlock(sx);
return (0);
} else {
sx_sunlock(sx);
return (1);
}
}
#ifdef KDTRACE_HOOKS
int
owner_sx(const struct lock_object *lock, struct thread **owner)
{
const struct sx *sx;
uintptr_t x;
sx = (const struct sx *)lock;
x = sx->sx_lock;
*owner = NULL;
return ((x & SX_LOCK_SHARED) != 0 ? (SX_SHARERS(x) != 0) :
((*owner = (struct thread *)SX_OWNER(x)) != NULL));
}
#endif
void
sx_sysinit(void *arg)
{
struct sx_args *sargs = arg;
sx_init_flags(sargs->sa_sx, sargs->sa_desc, sargs->sa_flags);
}
void
sx_init_flags(struct sx *sx, const char *description, int opts)
{
int flags;
MPASS((opts & ~(SX_QUIET | SX_RECURSE | SX_NOWITNESS | SX_DUPOK |
SX_NOPROFILE | SX_NEW)) == 0);
ASSERT_ATOMIC_LOAD_PTR(sx->sx_lock,
("%s: sx_lock not aligned for %s: %p", __func__, description,
&sx->sx_lock));
flags = LO_SLEEPABLE | LO_UPGRADABLE;
if (opts & SX_DUPOK)
flags |= LO_DUPOK;
if (opts & SX_NOPROFILE)
flags |= LO_NOPROFILE;
if (!(opts & SX_NOWITNESS))
flags |= LO_WITNESS;
if (opts & SX_RECURSE)
flags |= LO_RECURSABLE;
if (opts & SX_QUIET)
flags |= LO_QUIET;
if (opts & SX_NEW)
flags |= LO_NEW;
lock_init(&sx->lock_object, &lock_class_sx, description, NULL, flags);
sx->sx_lock = SX_LOCK_UNLOCKED;
sx->sx_recurse = 0;
}
void
sx_destroy(struct sx *sx)
{
KASSERT(sx->sx_lock == SX_LOCK_UNLOCKED, ("sx lock still held"));
KASSERT(sx->sx_recurse == 0, ("sx lock still recursed"));
sx->sx_lock = SX_LOCK_DESTROYED;
lock_destroy(&sx->lock_object);
}
int
sx_try_slock_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF)
{
uintptr_t x;
if (SCHEDULER_STOPPED())
return (1);
KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
("sx_try_slock() by idle thread %p on sx %s @ %s:%d",
curthread, sx->lock_object.lo_name, file, line));
x = sx->sx_lock;
for (;;) {
KASSERT(x != SX_LOCK_DESTROYED,
("sx_try_slock() of destroyed sx @ %s:%d", file, line));
if (!(x & SX_LOCK_SHARED))
break;
if (atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, x + SX_ONE_SHARER)) {
LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line);
WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line);
LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire,
sx, 0, 0, file, line, LOCKSTAT_READER);
TD_LOCKS_INC(curthread);
curthread->td_sx_slocks++;
return (1);
}
}
LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 0, file, line);
return (0);
}
int
sx_try_slock_(struct sx *sx, const char *file, int line)
{
return (sx_try_slock_int(sx LOCK_FILE_LINE_ARG));
}
int
_sx_xlock(struct sx *sx, int opts, const char *file, int line)
{
uintptr_t tid, x;
int error = 0;
KASSERT(kdb_active != 0 || SCHEDULER_STOPPED() ||
!TD_IS_IDLETHREAD(curthread),
("sx_xlock() by idle thread %p on sx %s @ %s:%d",
curthread, sx->lock_object.lo_name, file, line));
KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
("sx_xlock() of destroyed sx @ %s:%d", file, line));
WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
line, NULL);
tid = (uintptr_t)curthread;
x = SX_LOCK_UNLOCKED;
if (!atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, tid))
error = _sx_xlock_hard(sx, x, opts LOCK_FILE_LINE_ARG);
else
LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx,
0, 0, file, line, LOCKSTAT_WRITER);
if (!error) {
LOCK_LOG_LOCK("XLOCK", &sx->lock_object, 0, sx->sx_recurse,
file, line);
WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line);
TD_LOCKS_INC(curthread);
}
return (error);
}
int
sx_try_xlock_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF)
{
struct thread *td;
uintptr_t tid, x;
int rval;
bool recursed;
td = curthread;
tid = (uintptr_t)td;
if (SCHEDULER_STOPPED_TD(td))
return (1);
KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(td),
("sx_try_xlock() by idle thread %p on sx %s @ %s:%d",
curthread, sx->lock_object.lo_name, file, line));
KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
("sx_try_xlock() of destroyed sx @ %s:%d", file, line));
rval = 1;
recursed = false;
x = SX_LOCK_UNLOCKED;
for (;;) {
if (atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, tid))
break;
if (x == SX_LOCK_UNLOCKED)
continue;
if (x == tid && (sx->lock_object.lo_flags & LO_RECURSABLE)) {
sx->sx_recurse++;
atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
break;
}
rval = 0;
break;
}
LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, rval, file, line);
if (rval) {
WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
file, line);
if (!recursed)
LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire,
sx, 0, 0, file, line, LOCKSTAT_WRITER);
TD_LOCKS_INC(curthread);
}
return (rval);
}
int
sx_try_xlock_(struct sx *sx, const char *file, int line)
{
return (sx_try_xlock_int(sx LOCK_FILE_LINE_ARG));
}
void
_sx_xunlock(struct sx *sx, const char *file, int line)
{
KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
("sx_xunlock() of destroyed sx @ %s:%d", file, line));
_sx_assert(sx, SA_XLOCKED, file, line);
WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line);
LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, sx->sx_recurse, file,
line);
#if LOCK_DEBUG > 0
_sx_xunlock_hard(sx, (uintptr_t)curthread, file, line);
#else
__sx_xunlock(sx, curthread, file, line);
#endif
TD_LOCKS_DEC(curthread);
}
/*
* Try to do a non-blocking upgrade from a shared lock to an exclusive lock.
* This will only succeed if this thread holds a single shared lock.
* Return 1 if if the upgrade succeed, 0 otherwise.
*/
int
sx_try_upgrade_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF)
{
uintptr_t x;
uintptr_t waiters;
int success;
if (SCHEDULER_STOPPED())
return (1);
KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
("sx_try_upgrade() of destroyed sx @ %s:%d", file, line));
_sx_assert(sx, SA_SLOCKED, file, line);
/*
* Try to switch from one shared lock to an exclusive lock. We need
* to maintain the SX_LOCK_EXCLUSIVE_WAITERS flag if set so that
* we will wake up the exclusive waiters when we drop the lock.
*/
success = 0;
x = SX_READ_VALUE(sx);
for (;;) {
if (SX_SHARERS(x) > 1)
break;
waiters = (x & SX_LOCK_WAITERS);
if (atomic_fcmpset_acq_ptr(&sx->sx_lock, &x,
(uintptr_t)curthread | waiters)) {
success = 1;
break;
}
}
LOCK_LOG_TRY("XUPGRADE", &sx->lock_object, 0, success, file, line);
if (success) {
curthread->td_sx_slocks--;
WITNESS_UPGRADE(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
file, line);
LOCKSTAT_RECORD0(sx__upgrade, sx);
}
return (success);
}
int
sx_try_upgrade_(struct sx *sx, const char *file, int line)
{
return (sx_try_upgrade_int(sx LOCK_FILE_LINE_ARG));
}
/*
* Downgrade an unrecursed exclusive lock into a single shared lock.
*/
void
sx_downgrade_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF)
{
uintptr_t x;
int wakeup_swapper;
if (SCHEDULER_STOPPED())
return;
KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
("sx_downgrade() of destroyed sx @ %s:%d", file, line));
_sx_assert(sx, SA_XLOCKED | SA_NOTRECURSED, file, line);
#ifndef INVARIANTS
if (sx_recursed(sx))
panic("downgrade of a recursed lock");
#endif
WITNESS_DOWNGRADE(&sx->lock_object, 0, file, line);
/*
* Try to switch from an exclusive lock with no shared waiters
* to one sharer with no shared waiters. If there are
* exclusive waiters, we don't need to lock the sleep queue so
* long as we preserve the flag. We do one quick try and if
* that fails we grab the sleepq lock to keep the flags from
* changing and do it the slow way.
*
* We have to lock the sleep queue if there are shared waiters
* so we can wake them up.
*/
x = sx->sx_lock;
if (!(x & SX_LOCK_SHARED_WAITERS) &&
atomic_cmpset_rel_ptr(&sx->sx_lock, x, SX_SHARERS_LOCK(1) |
(x & SX_LOCK_EXCLUSIVE_WAITERS)))
goto out;
/*
* Lock the sleep queue so we can read the waiters bits
* without any races and wakeup any shared waiters.
*/
sleepq_lock(&sx->lock_object);
/*
* Preserve SX_LOCK_EXCLUSIVE_WAITERS while downgraded to a single
* shared lock. If there are any shared waiters, wake them up.
*/
wakeup_swapper = 0;
x = sx->sx_lock;
atomic_store_rel_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) |
(x & SX_LOCK_EXCLUSIVE_WAITERS));
if (x & SX_LOCK_SHARED_WAITERS)
wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX,
0, SQ_SHARED_QUEUE);
sleepq_release(&sx->lock_object);
if (wakeup_swapper)
kick_proc0();
out:
curthread->td_sx_slocks++;
LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line);
LOCKSTAT_RECORD0(sx__downgrade, sx);
}
void
sx_downgrade_(struct sx *sx, const char *file, int line)
{
sx_downgrade_int(sx LOCK_FILE_LINE_ARG);
}
#ifdef ADAPTIVE_SX
static inline void
sx_drop_critical(uintptr_t x, bool *in_critical, int *extra_work)
{
if (x & SX_LOCK_WRITE_SPINNER)
return;
if (*in_critical) {
critical_exit();
*in_critical = false;
(*extra_work)--;
}
}
#else
#define sx_drop_critical(x, in_critical, extra_work) do { } while(0)
#endif
/*
* This function represents the so-called 'hard case' for sx_xlock
* operation. All 'easy case' failures are redirected to this. Note
* that ideally this would be a static function, but it needs to be
* accessible from at least sx.h.
*/
int
_sx_xlock_hard(struct sx *sx, uintptr_t x, int opts LOCK_FILE_LINE_ARG_DEF)
{
GIANT_DECLARE;
uintptr_t tid, setx;
#ifdef ADAPTIVE_SX
volatile struct thread *owner;
u_int i, n, spintries = 0;
enum { READERS, WRITER } sleep_reason = READERS;
bool in_critical = false;
#endif
#ifdef LOCK_PROFILING
uint64_t waittime = 0;
int contested = 0;
#endif
int error = 0;
#if defined(ADAPTIVE_SX) || defined(KDTRACE_HOOKS)
struct lock_delay_arg lda;
#endif
#ifdef KDTRACE_HOOKS
u_int sleep_cnt = 0;
int64_t sleep_time = 0;
int64_t all_time = 0;
#endif
#if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING)
uintptr_t state = 0;
int doing_lockprof = 0;
#endif
int extra_work = 0;
tid = (uintptr_t)curthread;
#ifdef KDTRACE_HOOKS
if (LOCKSTAT_PROFILE_ENABLED(sx__acquire)) {
while (x == SX_LOCK_UNLOCKED) {
if (atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, tid))
goto out_lockstat;
}
extra_work = 1;
doing_lockprof = 1;
all_time -= lockstat_nsecs(&sx->lock_object);
state = x;
}
#endif
#ifdef LOCK_PROFILING
extra_work = 1;
doing_lockprof = 1;
state = x;
#endif
if (SCHEDULER_STOPPED())
return (0);
#if defined(ADAPTIVE_SX)
lock_delay_arg_init(&lda, &sx_delay);
#elif defined(KDTRACE_HOOKS)
lock_delay_arg_init(&lda, NULL);
#endif
if (__predict_false(x == SX_LOCK_UNLOCKED))
x = SX_READ_VALUE(sx);
/* If we already hold an exclusive lock, then recurse. */
if (__predict_false(lv_sx_owner(x) == (struct thread *)tid)) {
KASSERT((sx->lock_object.lo_flags & LO_RECURSABLE) != 0,
("_sx_xlock_hard: recursed on non-recursive sx %s @ %s:%d\n",
sx->lock_object.lo_name, file, line));
sx->sx_recurse++;
atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p recursing", __func__, sx);
return (0);
}
if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__,
sx->lock_object.lo_name, (void *)sx->sx_lock, file, line);
#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);
#endif
lock_profile_obtain_lock_failed(&sx->lock_object, &contested,
&waittime);
#ifndef INVARIANTS
GIANT_SAVE(extra_work);
#endif
for (;;) {
if (x == SX_LOCK_UNLOCKED) {
if (atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, tid))
break;
continue;
}
#ifdef INVARIANTS
GIANT_SAVE(extra_work);
#endif
#ifdef KDTRACE_HOOKS
lda.spin_cnt++;
#endif
#ifdef ADAPTIVE_SX
if (x == (SX_LOCK_SHARED | SX_LOCK_WRITE_SPINNER)) {
if (atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, tid))
break;
continue;
}
/*
* If the lock is write locked and the owner is
* running on another CPU, spin until the owner stops
* running or the state of the lock changes.
*/
if ((x & SX_LOCK_SHARED) == 0) {
sx_drop_critical(x, &in_critical, &extra_work);
sleep_reason = WRITER;
owner = lv_sx_owner(x);
if (!TD_IS_RUNNING(owner))
goto sleepq;
if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR3(KTR_LOCK, "%s: spinning on %p held by %p",
__func__, sx, owner);
KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread),
"spinning", "lockname:\"%s\"",
sx->lock_object.lo_name);
do {
lock_delay(&lda);
x = SX_READ_VALUE(sx);
owner = lv_sx_owner(x);
} while (owner != NULL && TD_IS_RUNNING(owner));
KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread),
"running");
continue;
} else if (SX_SHARERS(x) > 0) {
sleep_reason = READERS;
if (spintries == asx_retries)
goto sleepq;
if (!(x & SX_LOCK_WRITE_SPINNER)) {
if (!in_critical) {
critical_enter();
in_critical = true;
extra_work++;
}
if (!atomic_fcmpset_ptr(&sx->sx_lock, &x,
x | SX_LOCK_WRITE_SPINNER)) {
critical_exit();
in_critical = false;
extra_work--;
continue;
}
}
spintries++;
KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread),
"spinning", "lockname:\"%s\"",
sx->lock_object.lo_name);
n = SX_SHARERS(x);
for (i = 0; i < asx_loops; i += n) {
lock_delay_spin(n);
x = SX_READ_VALUE(sx);
if (!(x & SX_LOCK_WRITE_SPINNER))
break;
if (!(x & SX_LOCK_SHARED))
break;
n = SX_SHARERS(x);
if (n == 0)
break;
}
#ifdef KDTRACE_HOOKS
lda.spin_cnt += i;
#endif
KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread),
"running");
if (i < asx_loops)
continue;
}
sleepq:
#endif
sleepq_lock(&sx->lock_object);
x = SX_READ_VALUE(sx);
retry_sleepq:
/*
* If the lock was released while spinning on the
* sleep queue chain lock, try again.
*/
if (x == SX_LOCK_UNLOCKED) {
sleepq_release(&sx->lock_object);
sx_drop_critical(x, &in_critical, &extra_work);
continue;
}
#ifdef ADAPTIVE_SX
/*
* The current lock owner might have started executing
* on another CPU (or the lock could have changed
* owners) while we were waiting on the sleep queue
* chain lock. If so, drop the sleep queue lock and try
* again.
*/
if (!(x & SX_LOCK_SHARED)) {
owner = (struct thread *)SX_OWNER(x);
if (TD_IS_RUNNING(owner)) {
sleepq_release(&sx->lock_object);
sx_drop_critical(x, &in_critical,
&extra_work);
continue;
}
} else if (SX_SHARERS(x) > 0 && sleep_reason == WRITER) {
sleepq_release(&sx->lock_object);
sx_drop_critical(x, &in_critical, &extra_work);
continue;
}
#endif
/*
* If an exclusive lock was released with both shared
* and exclusive waiters and a shared waiter hasn't
* woken up and acquired the lock yet, sx_lock will be
* set to SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS.
* If we see that value, try to acquire it once. Note
* that we have to preserve SX_LOCK_EXCLUSIVE_WAITERS
* as there are other exclusive waiters still. If we
* fail, restart the loop.
*/
setx = x & (SX_LOCK_WAITERS | SX_LOCK_WRITE_SPINNER);
if ((x & ~setx) == SX_LOCK_SHARED) {
setx &= ~SX_LOCK_WRITE_SPINNER;
if (!atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, tid | setx))
goto retry_sleepq;
sleepq_release(&sx->lock_object);
CTR2(KTR_LOCK, "%s: %p claimed by new writer",
__func__, sx);
break;
}
#ifdef ADAPTIVE_SX
/*
* It is possible we set the SX_LOCK_WRITE_SPINNER bit.
* It is an invariant that when the bit is set, there is
* a writer ready to grab the lock. Thus clear the bit since
* we are going to sleep.
*/
if (in_critical) {
if ((x & SX_LOCK_WRITE_SPINNER) ||
!((x & SX_LOCK_EXCLUSIVE_WAITERS))) {
setx = x & ~SX_LOCK_WRITE_SPINNER;
setx |= SX_LOCK_EXCLUSIVE_WAITERS;
if (!atomic_fcmpset_ptr(&sx->sx_lock, &x,
setx)) {
goto retry_sleepq;
}
}
critical_exit();
in_critical = false;
} else {
#endif
/*
* Try to set the SX_LOCK_EXCLUSIVE_WAITERS. If we fail,
* than loop back and retry.
*/
if (!(x & SX_LOCK_EXCLUSIVE_WAITERS)) {
if (!atomic_fcmpset_ptr(&sx->sx_lock, &x,
x | SX_LOCK_EXCLUSIVE_WAITERS)) {
goto retry_sleepq;
}
if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p set excl waiters flag",
__func__, sx);
}
#ifdef ADAPTIVE_SX
}
#endif
/*
* Since we have been unable to acquire the exclusive
* lock and the exclusive waiters flag is set, we have
* to sleep.
*/
if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p blocking on sleep queue",
__func__, sx);
#ifdef KDTRACE_HOOKS
sleep_time -= lockstat_nsecs(&sx->lock_object);
#endif
sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name,
SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ?
SLEEPQ_INTERRUPTIBLE : 0), SQ_EXCLUSIVE_QUEUE);
if (!(opts & SX_INTERRUPTIBLE))
sleepq_wait(&sx->lock_object, 0);
else
error = sleepq_wait_sig(&sx->lock_object, 0);
#ifdef KDTRACE_HOOKS
sleep_time += lockstat_nsecs(&sx->lock_object);
sleep_cnt++;
#endif
if (error) {
if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR2(KTR_LOCK,
"%s: interruptible sleep by %p suspended by signal",
__func__, sx);
break;
}
if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p resuming from sleep queue",
__func__, sx);
x = SX_READ_VALUE(sx);
}
if (__predict_true(!extra_work))
return (error);
#ifdef ADAPTIVE_SX
if (in_critical)
critical_exit();
#endif
GIANT_RESTORE();
#if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING)
if (__predict_true(!doing_lockprof))
return (error);
#endif
#ifdef KDTRACE_HOOKS
all_time += lockstat_nsecs(&sx->lock_object);
if (sleep_time)
LOCKSTAT_RECORD4(sx__block, sx, sleep_time,
LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0,
(state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
if (lda.spin_cnt > sleep_cnt)
LOCKSTAT_RECORD4(sx__spin, sx, all_time - sleep_time,
LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0,
(state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
out_lockstat:
#endif
if (!error)
LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx,
contested, waittime, file, line, LOCKSTAT_WRITER);
return (error);
}
/*
* This function represents the so-called 'hard case' for sx_xunlock
* operation. All 'easy case' failures are redirected to this. Note
* that ideally this would be a static function, but it needs to be
* accessible from at least sx.h.
*/
void
_sx_xunlock_hard(struct sx *sx, uintptr_t x LOCK_FILE_LINE_ARG_DEF)
{
uintptr_t tid, setx;
int queue, wakeup_swapper;
if (SCHEDULER_STOPPED())
return;
tid = (uintptr_t)curthread;
if (__predict_false(x == tid))
x = SX_READ_VALUE(sx);
MPASS(!(x & SX_LOCK_SHARED));
if (__predict_false(x & SX_LOCK_RECURSED)) {
/* The lock is recursed, unrecurse one level. */
if ((--sx->sx_recurse) == 0)
atomic_clear_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, sx);
return;
}
LOCKSTAT_PROFILE_RELEASE_RWLOCK(sx__release, sx, LOCKSTAT_WRITER);
if (x == tid &&
atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_LOCK_UNLOCKED))
return;
if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p contested", __func__, sx);
sleepq_lock(&sx->lock_object);
x = SX_READ_VALUE(sx);
MPASS(x & (SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS));
/*
* The wake up algorithm here is quite simple and probably not
* ideal. It gives precedence to shared waiters if they are
* present. For this condition, we have to preserve the
* state of the exclusive waiters flag.
* If interruptible sleeps left the shared queue empty avoid a
* starvation for the threads sleeping on the exclusive queue by giving
* them precedence and cleaning up the shared waiters bit anyway.
*/
setx = SX_LOCK_UNLOCKED;
queue = SQ_SHARED_QUEUE;
if ((x & SX_LOCK_EXCLUSIVE_WAITERS) != 0 &&
sleepq_sleepcnt(&sx->lock_object, SQ_EXCLUSIVE_QUEUE) != 0) {
queue = SQ_EXCLUSIVE_QUEUE;
setx |= (x & SX_LOCK_SHARED_WAITERS);
}
atomic_store_rel_ptr(&sx->sx_lock, setx);
/* Wake up all the waiters for the specific queue. */
if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR3(KTR_LOCK, "%s: %p waking up all threads on %s queue",
__func__, sx, queue == SQ_SHARED_QUEUE ? "shared" :
"exclusive");
wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0,
queue);
sleepq_release(&sx->lock_object);
if (wakeup_swapper)
kick_proc0();
}
static bool __always_inline
__sx_can_read(struct thread *td, uintptr_t x, bool fp)
{
if ((x & (SX_LOCK_SHARED | SX_LOCK_EXCLUSIVE_WAITERS | SX_LOCK_WRITE_SPINNER))
== SX_LOCK_SHARED)
return (true);
if (!fp && td->td_sx_slocks && (x & SX_LOCK_SHARED))
return (true);
return (false);
}
static bool __always_inline
__sx_slock_try(struct sx *sx, struct thread *td, uintptr_t *xp, bool fp
LOCK_FILE_LINE_ARG_DEF)
{
/*
* If no other thread has an exclusive lock then try to bump up
* the count of sharers. Since we have to preserve the state
* of SX_LOCK_EXCLUSIVE_WAITERS, if we fail to acquire the
* shared lock loop back and retry.
*/
while (__sx_can_read(td, *xp, fp)) {
if (atomic_fcmpset_acq_ptr(&sx->sx_lock, xp,
*xp + SX_ONE_SHARER)) {
if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR4(KTR_LOCK, "%s: %p succeed %p -> %p",
__func__, sx, (void *)*xp,
(void *)(*xp + SX_ONE_SHARER));
td->td_sx_slocks++;
return (true);
}
}
return (false);
}
static int __noinline
_sx_slock_hard(struct sx *sx, int opts, uintptr_t x LOCK_FILE_LINE_ARG_DEF)
{
GIANT_DECLARE;
struct thread *td;
#ifdef ADAPTIVE_SX
volatile struct thread *owner;
u_int i, n, spintries = 0;
#endif
#ifdef LOCK_PROFILING
uint64_t waittime = 0;
int contested = 0;
#endif
int error = 0;
#if defined(ADAPTIVE_SX) || defined(KDTRACE_HOOKS)
struct lock_delay_arg lda;
#endif
#ifdef KDTRACE_HOOKS
u_int sleep_cnt = 0;
int64_t sleep_time = 0;
int64_t all_time = 0;
#endif
#if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING)
uintptr_t state = 0;
#endif
int extra_work = 0;
td = curthread;
#ifdef KDTRACE_HOOKS
if (LOCKSTAT_PROFILE_ENABLED(sx__acquire)) {
if (__sx_slock_try(sx, td, &x, false LOCK_FILE_LINE_ARG))
goto out_lockstat;
extra_work = 1;
all_time -= lockstat_nsecs(&sx->lock_object);
state = x;
}
#endif
#ifdef LOCK_PROFILING
extra_work = 1;
state = x;
#endif
if (SCHEDULER_STOPPED())
return (0);
#if defined(ADAPTIVE_SX)
lock_delay_arg_init(&lda, &sx_delay);
#elif defined(KDTRACE_HOOKS)
lock_delay_arg_init(&lda, NULL);
#endif
#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);
#endif
lock_profile_obtain_lock_failed(&sx->lock_object, &contested,
&waittime);
#ifndef INVARIANTS
GIANT_SAVE(extra_work);
#endif
/*
* As with rwlocks, we don't make any attempt to try to block
* shared locks once there is an exclusive waiter.
*/
for (;;) {
if (__sx_slock_try(sx, td, &x, false LOCK_FILE_LINE_ARG))
break;
#ifdef INVARIANTS
GIANT_SAVE(extra_work);
#endif
#ifdef KDTRACE_HOOKS
lda.spin_cnt++;
#endif
#ifdef ADAPTIVE_SX
/*
* If the owner is running on another CPU, spin until
* the owner stops running or the state of the lock
* changes.
*/
if ((x & SX_LOCK_SHARED) == 0) {
owner = lv_sx_owner(x);
if (TD_IS_RUNNING(owner)) {
if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR3(KTR_LOCK,
"%s: spinning on %p held by %p",
__func__, sx, owner);
KTR_STATE1(KTR_SCHED, "thread",
sched_tdname(curthread), "spinning",
"lockname:\"%s\"", sx->lock_object.lo_name);
do {
lock_delay(&lda);
x = SX_READ_VALUE(sx);
owner = lv_sx_owner(x);
} while (owner != NULL && TD_IS_RUNNING(owner));
KTR_STATE0(KTR_SCHED, "thread",
sched_tdname(curthread), "running");
continue;
}
} else {
if ((x & SX_LOCK_WRITE_SPINNER) && SX_SHARERS(x) == 0) {
MPASS(!__sx_can_read(td, x, false));
lock_delay_spin(2);
x = SX_READ_VALUE(sx);
continue;
}
if (spintries < asx_retries) {
KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread),
"spinning", "lockname:\"%s\"",
sx->lock_object.lo_name);
n = SX_SHARERS(x);
for (i = 0; i < asx_loops; i += n) {
lock_delay_spin(n);
x = SX_READ_VALUE(sx);
if (!(x & SX_LOCK_SHARED))
break;
n = SX_SHARERS(x);
if (n == 0)
break;
if (__sx_can_read(td, x, false))
break;
}
#ifdef KDTRACE_HOOKS
lda.spin_cnt += i;
#endif
KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread),
"running");
if (i < asx_loops)
continue;
}
}
#endif
/*
* Some other thread already has an exclusive lock, so
* start the process of blocking.
*/
sleepq_lock(&sx->lock_object);
x = SX_READ_VALUE(sx);
retry_sleepq:
if (((x & SX_LOCK_WRITE_SPINNER) && SX_SHARERS(x) == 0) ||
__sx_can_read(td, x, false)) {
sleepq_release(&sx->lock_object);
continue;
}
#ifdef ADAPTIVE_SX
/*
* If the owner is running on another CPU, spin until
* the owner stops running or the state of the lock
* changes.
*/
if (!(x & SX_LOCK_SHARED)) {
owner = (struct thread *)SX_OWNER(x);
if (TD_IS_RUNNING(owner)) {
sleepq_release(&sx->lock_object);
x = SX_READ_VALUE(sx);
continue;
}
}
#endif
/*
* Try to set the SX_LOCK_SHARED_WAITERS flag. If we
* fail to set it drop the sleep queue lock and loop
* back.
*/
if (!(x & SX_LOCK_SHARED_WAITERS)) {
if (!atomic_fcmpset_ptr(&sx->sx_lock, &x,
x | SX_LOCK_SHARED_WAITERS))
goto retry_sleepq;
if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p set shared waiters flag",
__func__, sx);
}
/*
* Since we have been unable to acquire the shared lock,
* we have to sleep.
*/
if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p blocking on sleep queue",
__func__, sx);
#ifdef KDTRACE_HOOKS
sleep_time -= lockstat_nsecs(&sx->lock_object);
#endif
sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name,
SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ?
SLEEPQ_INTERRUPTIBLE : 0), SQ_SHARED_QUEUE);
if (!(opts & SX_INTERRUPTIBLE))
sleepq_wait(&sx->lock_object, 0);
else
error = sleepq_wait_sig(&sx->lock_object, 0);
#ifdef KDTRACE_HOOKS
sleep_time += lockstat_nsecs(&sx->lock_object);
sleep_cnt++;
#endif
if (error) {
if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR2(KTR_LOCK,
"%s: interruptible sleep by %p suspended by signal",
__func__, sx);
break;
}
if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p resuming from sleep queue",
__func__, sx);
x = SX_READ_VALUE(sx);
}
#if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING)
if (__predict_true(!extra_work))
return (error);
#endif
#ifdef KDTRACE_HOOKS
all_time += lockstat_nsecs(&sx->lock_object);
if (sleep_time)
LOCKSTAT_RECORD4(sx__block, sx, sleep_time,
LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0,
(state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
if (lda.spin_cnt > sleep_cnt)
LOCKSTAT_RECORD4(sx__spin, sx, all_time - sleep_time,
LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0,
(state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
out_lockstat:
#endif
if (error == 0) {
LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx,
contested, waittime, file, line, LOCKSTAT_READER);
}
GIANT_RESTORE();
return (error);
}
int
_sx_slock_int(struct sx *sx, int opts LOCK_FILE_LINE_ARG_DEF)
{
struct thread *td;
uintptr_t x;
int error;
KASSERT(kdb_active != 0 || SCHEDULER_STOPPED() ||
!TD_IS_IDLETHREAD(curthread),
("sx_slock() by idle thread %p on sx %s @ %s:%d",
curthread, sx->lock_object.lo_name, file, line));
KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
("sx_slock() of destroyed sx @ %s:%d", file, line));
WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line, NULL);
error = 0;
td = curthread;
x = SX_READ_VALUE(sx);
if (__predict_false(LOCKSTAT_PROFILE_ENABLED(sx__acquire) ||
!__sx_slock_try(sx, td, &x, true LOCK_FILE_LINE_ARG)))
error = _sx_slock_hard(sx, opts, x LOCK_FILE_LINE_ARG);
else
lock_profile_obtain_lock_success(&sx->lock_object, 0, 0,
file, line);
if (error == 0) {
LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line);
WITNESS_LOCK(&sx->lock_object, 0, file, line);
TD_LOCKS_INC(curthread);
}
return (error);
}
int
_sx_slock(struct sx *sx, int opts, const char *file, int line)
{
return (_sx_slock_int(sx, opts LOCK_FILE_LINE_ARG));
}
static bool __always_inline
_sx_sunlock_try(struct sx *sx, struct thread *td, uintptr_t *xp)
{
for (;;) {
if (SX_SHARERS(*xp) > 1 || !(*xp & SX_LOCK_WAITERS)) {
if (atomic_fcmpset_rel_ptr(&sx->sx_lock, xp,
*xp - SX_ONE_SHARER)) {
if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR4(KTR_LOCK,
"%s: %p succeeded %p -> %p",
__func__, sx, (void *)*xp,
(void *)(*xp - SX_ONE_SHARER));
td->td_sx_slocks--;
return (true);
}
continue;
}
break;
}
return (false);
}
static void __noinline
_sx_sunlock_hard(struct sx *sx, struct thread *td, uintptr_t x
LOCK_FILE_LINE_ARG_DEF)
{
int wakeup_swapper = 0;
uintptr_t setx, queue;
if (SCHEDULER_STOPPED())
return;
if (_sx_sunlock_try(sx, td, &x))
goto out_lockstat;
sleepq_lock(&sx->lock_object);
x = SX_READ_VALUE(sx);
for (;;) {
if (_sx_sunlock_try(sx, td, &x))
break;
/*
* Wake up semantic here is quite simple:
* Just wake up all the exclusive waiters.
* Note that the state of the lock could have changed,
* so if it fails loop back and retry.
*/
setx = SX_LOCK_UNLOCKED;
queue = SQ_SHARED_QUEUE;
if (x & SX_LOCK_EXCLUSIVE_WAITERS) {
setx |= (x & SX_LOCK_SHARED_WAITERS);
queue = SQ_EXCLUSIVE_QUEUE;
}
setx |= (x & SX_LOCK_WRITE_SPINNER);
if (!atomic_fcmpset_rel_ptr(&sx->sx_lock, &x, setx))
continue;
if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p waking up all thread on"
"exclusive queue", __func__, sx);
wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX,
0, queue);
td->td_sx_slocks--;
break;
}
sleepq_release(&sx->lock_object);
if (wakeup_swapper)
kick_proc0();
out_lockstat:
LOCKSTAT_PROFILE_RELEASE_RWLOCK(sx__release, sx, LOCKSTAT_READER);
}
void
_sx_sunlock_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF)
{
struct thread *td;
uintptr_t x;
KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
("sx_sunlock() of destroyed sx @ %s:%d", file, line));
_sx_assert(sx, SA_SLOCKED, file, line);
WITNESS_UNLOCK(&sx->lock_object, 0, file, line);
LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line);
td = curthread;
x = SX_READ_VALUE(sx);
if (__predict_false(LOCKSTAT_PROFILE_ENABLED(sx__release) ||
!_sx_sunlock_try(sx, td, &x)))
_sx_sunlock_hard(sx, td, x LOCK_FILE_LINE_ARG);
else
lock_profile_release_lock(&sx->lock_object);
TD_LOCKS_DEC(curthread);
}
void
_sx_sunlock(struct sx *sx, const char *file, int line)
{
_sx_sunlock_int(sx LOCK_FILE_LINE_ARG);
}
#ifdef INVARIANT_SUPPORT
#ifndef INVARIANTS
#undef _sx_assert
#endif
/*
* In the non-WITNESS case, sx_assert() can only detect that at least
* *some* thread owns an slock, but it cannot guarantee that *this*
* thread owns an slock.
*/
void
_sx_assert(const struct sx *sx, int what, const char *file, int line)
{
#ifndef WITNESS
int slocked = 0;
#endif
if (SCHEDULER_STOPPED())
return;
switch (what) {
case SA_SLOCKED:
case SA_SLOCKED | SA_NOTRECURSED:
case SA_SLOCKED | SA_RECURSED:
#ifndef WITNESS
slocked = 1;
/* FALLTHROUGH */
#endif
case SA_LOCKED:
case SA_LOCKED | SA_NOTRECURSED:
case SA_LOCKED | SA_RECURSED:
#ifdef WITNESS
witness_assert(&sx->lock_object, what, file, line);
#else
/*
* If some other thread has an exclusive lock or we
* have one and are asserting a shared lock, fail.
* Also, if no one has a lock at all, fail.
*/
if (sx->sx_lock == SX_LOCK_UNLOCKED ||
(!(sx->sx_lock & SX_LOCK_SHARED) && (slocked ||
sx_xholder(sx) != curthread)))
panic("Lock %s not %slocked @ %s:%d\n",
sx->lock_object.lo_name, slocked ? "share " : "",
file, line);
if (!(sx->sx_lock & SX_LOCK_SHARED)) {
if (sx_recursed(sx)) {
if (what & SA_NOTRECURSED)
panic("Lock %s recursed @ %s:%d\n",
sx->lock_object.lo_name, file,
line);
} else if (what & SA_RECURSED)
panic("Lock %s not recursed @ %s:%d\n",
sx->lock_object.lo_name, file, line);
}
#endif
break;
case SA_XLOCKED:
case SA_XLOCKED | SA_NOTRECURSED:
case SA_XLOCKED | SA_RECURSED:
if (sx_xholder(sx) != curthread)
panic("Lock %s not exclusively locked @ %s:%d\n",
sx->lock_object.lo_name, file, line);
if (sx_recursed(sx)) {
if (what & SA_NOTRECURSED)
panic("Lock %s recursed @ %s:%d\n",
sx->lock_object.lo_name, file, line);
} else if (what & SA_RECURSED)
panic("Lock %s not recursed @ %s:%d\n",
sx->lock_object.lo_name, file, line);
break;
case SA_UNLOCKED:
#ifdef WITNESS
witness_assert(&sx->lock_object, what, file, line);
#else
/*
* If we hold an exclusve lock fail. We can't
* reliably check to see if we hold a shared lock or
* not.
*/
if (sx_xholder(sx) == curthread)
panic("Lock %s exclusively locked @ %s:%d\n",
sx->lock_object.lo_name, file, line);
#endif
break;
default:
panic("Unknown sx lock assertion: %d @ %s:%d", what, file,
line);
}
}
#endif /* INVARIANT_SUPPORT */
#ifdef DDB
static void
db_show_sx(const struct lock_object *lock)
{
struct thread *td;
const struct sx *sx;
sx = (const struct sx *)lock;
db_printf(" state: ");
if (sx->sx_lock == SX_LOCK_UNLOCKED)
db_printf("UNLOCKED\n");
else if (sx->sx_lock == SX_LOCK_DESTROYED) {
db_printf("DESTROYED\n");
return;
} else if (sx->sx_lock & SX_LOCK_SHARED)
db_printf("SLOCK: %ju\n", (uintmax_t)SX_SHARERS(sx->sx_lock));
else {
td = sx_xholder(sx);
db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
td->td_tid, td->td_proc->p_pid, td->td_name);
if (sx_recursed(sx))
db_printf(" recursed: %d\n", sx->sx_recurse);
}
db_printf(" waiters: ");
switch(sx->sx_lock &
(SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS)) {
case SX_LOCK_SHARED_WAITERS:
db_printf("shared\n");
break;
case SX_LOCK_EXCLUSIVE_WAITERS:
db_printf("exclusive\n");
break;
case SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS:
db_printf("exclusive and shared\n");
break;
default:
db_printf("none\n");
}
}
/*
* Check to see if a thread that is blocked on a sleep queue is actually
* blocked on an sx lock. If so, output some details and return true.
* If the lock has an exclusive owner, return that in *ownerp.
*/
int
sx_chain(struct thread *td, struct thread **ownerp)
{
- struct sx *sx;
+ const struct sx *sx;
/*
* Check to see if this thread is blocked on an sx lock.
* First, we check the lock class. If that is ok, then we
* compare the lock name against the wait message.
*/
sx = td->td_wchan;
if (LOCK_CLASS(&sx->lock_object) != &lock_class_sx ||
sx->lock_object.lo_name != td->td_wmesg)
return (0);
/* We think we have an sx lock, so output some details. */
db_printf("blocked on sx \"%s\" ", td->td_wmesg);
*ownerp = sx_xholder(sx);
if (sx->sx_lock & SX_LOCK_SHARED)
db_printf("SLOCK (count %ju)\n",
(uintmax_t)SX_SHARERS(sx->sx_lock));
else
db_printf("XLOCK\n");
return (1);
}
#endif
Index: head/sys/kern/kern_synch.c
===================================================================
--- head/sys/kern/kern_synch.c (revision 356056)
+++ head/sys/kern/kern_synch.c (revision 356057)
@@ -1,668 +1,668 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1982, 1986, 1990, 1991, 1993
* The Regents of the University of California. All rights reserved.
* (c) UNIX System Laboratories, Inc.
* All or some portions of this file are derived from material licensed
* to the University of California by American Telephone and Telegraph
* Co. or Unix System Laboratories, Inc. and are reproduced herein with
* the permission of UNIX System Laboratories, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)kern_synch.c 8.9 (Berkeley) 5/19/95
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_ktrace.h"
#include "opt_sched.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/condvar.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/resourcevar.h>
#include <sys/refcount.h>
#include <sys/sched.h>
#include <sys/sdt.h>
#include <sys/signalvar.h>
#include <sys/sleepqueue.h>
#include <sys/smp.h>
#include <sys/sx.h>
#include <sys/sysctl.h>
#include <sys/sysproto.h>
#include <sys/vmmeter.h>
#ifdef KTRACE
#include <sys/uio.h>
#include <sys/ktrace.h>
#endif
#ifdef EPOCH_TRACE
#include <sys/epoch.h>
#endif
#include <machine/cpu.h>
static void synch_setup(void *dummy);
SYSINIT(synch_setup, SI_SUB_KICK_SCHEDULER, SI_ORDER_FIRST, synch_setup,
NULL);
int hogticks;
-static char pause_wchan[MAXCPU];
+static const char pause_wchan[MAXCPU];
static struct callout loadav_callout;
struct loadavg averunnable =
{ {0, 0, 0}, FSCALE }; /* load average, of runnable procs */
/*
* Constants for averages over 1, 5, and 15 minutes
* when sampling at 5 second intervals.
*/
static fixpt_t cexp[3] = {
0.9200444146293232 * FSCALE, /* exp(-1/12) */
0.9834714538216174 * FSCALE, /* exp(-1/60) */
0.9944598480048967 * FSCALE, /* exp(-1/180) */
};
/* kernel uses `FSCALE', userland (SHOULD) use kern.fscale */
SYSCTL_INT(_kern, OID_AUTO, fscale, CTLFLAG_RD, SYSCTL_NULL_INT_PTR, FSCALE, "");
static void loadav(void *arg);
SDT_PROVIDER_DECLARE(sched);
SDT_PROBE_DEFINE(sched, , , preempt);
static void
sleepinit(void *unused)
{
hogticks = (hz / 10) * 2; /* Default only. */
init_sleepqueues();
}
/*
* vmem tries to lock the sleepq mutexes when free'ing kva, so make sure
* it is available.
*/
SYSINIT(sleepinit, SI_SUB_KMEM, SI_ORDER_ANY, sleepinit, NULL);
/*
* General sleep call. Suspends the current thread until a wakeup is
* performed on the specified identifier. The thread will then be made
* runnable with the specified priority. Sleeps at most sbt units of time
* (0 means no timeout). If pri includes the PCATCH flag, let signals
* interrupt the sleep, otherwise ignore them while sleeping. Returns 0 if
* awakened, EWOULDBLOCK if the timeout expires. If PCATCH is set and a
* signal becomes pending, ERESTART is returned if the current system
* call should be restarted if possible, and EINTR is returned if the system
* call should be interrupted by the signal (return EINTR).
*
* The lock argument is unlocked before the caller is suspended, and
* re-locked before _sleep() returns. If priority includes the PDROP
* flag the lock is not re-locked before returning.
*/
int
-_sleep(void *ident, struct lock_object *lock, int priority,
+_sleep(const void *ident, struct lock_object *lock, int priority,
const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags)
{
struct thread *td;
struct lock_class *class;
uintptr_t lock_state;
int catch, pri, rval, sleepq_flags;
WITNESS_SAVE_DECL(lock_witness);
td = curthread;
#ifdef KTRACE
if (KTRPOINT(td, KTR_CSW))
ktrcsw(1, 0, wmesg);
#endif
WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
"Sleeping on \"%s\"", wmesg);
KASSERT(sbt != 0 || mtx_owned(&Giant) || lock != NULL,
("sleeping without a lock"));
KASSERT(ident != NULL, ("_sleep: NULL ident"));
KASSERT(TD_IS_RUNNING(td), ("_sleep: curthread not running"));
if (priority & PDROP)
KASSERT(lock != NULL && lock != &Giant.lock_object,
("PDROP requires a non-Giant lock"));
if (lock != NULL)
class = LOCK_CLASS(lock);
else
class = NULL;
if (SCHEDULER_STOPPED_TD(td)) {
if (lock != NULL && priority & PDROP)
class->lc_unlock(lock);
return (0);
}
catch = priority & PCATCH;
pri = priority & PRIMASK;
KASSERT(!TD_ON_SLEEPQ(td), ("recursive sleep"));
if ((uintptr_t)ident >= (uintptr_t)&pause_wchan[0] &&
(uintptr_t)ident <= (uintptr_t)&pause_wchan[MAXCPU - 1])
sleepq_flags = SLEEPQ_PAUSE;
else
sleepq_flags = SLEEPQ_SLEEP;
if (catch)
sleepq_flags |= SLEEPQ_INTERRUPTIBLE;
sleepq_lock(ident);
CTR5(KTR_PROC, "sleep: thread %ld (pid %ld, %s) on %s (%p)",
td->td_tid, td->td_proc->p_pid, td->td_name, wmesg, ident);
if (lock == &Giant.lock_object)
mtx_assert(&Giant, MA_OWNED);
DROP_GIANT();
if (lock != NULL && lock != &Giant.lock_object &&
!(class->lc_flags & LC_SLEEPABLE)) {
WITNESS_SAVE(lock, lock_witness);
lock_state = class->lc_unlock(lock);
} else
/* GCC needs to follow the Yellow Brick Road */
lock_state = -1;
/*
* We put ourselves on the sleep queue and start our timeout
* before calling thread_suspend_check, as we could stop there,
* and a wakeup or a SIGCONT (or both) could occur while we were
* stopped without resuming us. Thus, we must be ready for sleep
* when cursig() is called. If the wakeup happens while we're
* stopped, then td will no longer be on a sleep queue upon
* return from cursig().
*/
sleepq_add(ident, lock, wmesg, sleepq_flags, 0);
if (sbt != 0)
sleepq_set_timeout_sbt(ident, sbt, pr, flags);
if (lock != NULL && class->lc_flags & LC_SLEEPABLE) {
sleepq_release(ident);
WITNESS_SAVE(lock, lock_witness);
lock_state = class->lc_unlock(lock);
sleepq_lock(ident);
}
if (sbt != 0 && catch)
rval = sleepq_timedwait_sig(ident, pri);
else if (sbt != 0)
rval = sleepq_timedwait(ident, pri);
else if (catch)
rval = sleepq_wait_sig(ident, pri);
else {
sleepq_wait(ident, pri);
rval = 0;
}
#ifdef KTRACE
if (KTRPOINT(td, KTR_CSW))
ktrcsw(0, 0, wmesg);
#endif
PICKUP_GIANT();
if (lock != NULL && lock != &Giant.lock_object && !(priority & PDROP)) {
class->lc_lock(lock, lock_state);
WITNESS_RESTORE(lock, lock_witness);
}
return (rval);
}
int
-msleep_spin_sbt(void *ident, struct mtx *mtx, const char *wmesg,
+msleep_spin_sbt(const void *ident, struct mtx *mtx, const char *wmesg,
sbintime_t sbt, sbintime_t pr, int flags)
{
struct thread *td;
int rval;
WITNESS_SAVE_DECL(mtx);
td = curthread;
KASSERT(mtx != NULL, ("sleeping without a mutex"));
KASSERT(ident != NULL, ("msleep_spin_sbt: NULL ident"));
KASSERT(TD_IS_RUNNING(td), ("msleep_spin_sbt: curthread not running"));
if (SCHEDULER_STOPPED_TD(td))
return (0);
sleepq_lock(ident);
CTR5(KTR_PROC, "msleep_spin: thread %ld (pid %ld, %s) on %s (%p)",
td->td_tid, td->td_proc->p_pid, td->td_name, wmesg, ident);
DROP_GIANT();
mtx_assert(mtx, MA_OWNED | MA_NOTRECURSED);
WITNESS_SAVE(&mtx->lock_object, mtx);
mtx_unlock_spin(mtx);
/*
* We put ourselves on the sleep queue and start our timeout.
*/
sleepq_add(ident, &mtx->lock_object, wmesg, SLEEPQ_SLEEP, 0);
if (sbt != 0)
sleepq_set_timeout_sbt(ident, sbt, pr, flags);
/*
* Can't call ktrace with any spin locks held so it can lock the
* ktrace_mtx lock, and WITNESS_WARN considers it an error to hold
* any spin lock. Thus, we have to drop the sleepq spin lock while
* we handle those requests. This is safe since we have placed our
* thread on the sleep queue already.
*/
#ifdef KTRACE
if (KTRPOINT(td, KTR_CSW)) {
sleepq_release(ident);
ktrcsw(1, 0, wmesg);
sleepq_lock(ident);
}
#endif
#ifdef WITNESS
sleepq_release(ident);
WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "Sleeping on \"%s\"",
wmesg);
sleepq_lock(ident);
#endif
if (sbt != 0)
rval = sleepq_timedwait(ident, 0);
else {
sleepq_wait(ident, 0);
rval = 0;
}
#ifdef KTRACE
if (KTRPOINT(td, KTR_CSW))
ktrcsw(0, 0, wmesg);
#endif
PICKUP_GIANT();
mtx_lock_spin(mtx);
WITNESS_RESTORE(&mtx->lock_object, mtx);
return (rval);
}
/*
* pause_sbt() delays the calling thread by the given signed binary
* time. During cold bootup, pause_sbt() uses the DELAY() function
* instead of the _sleep() function to do the waiting. The "sbt"
* argument must be greater than or equal to zero. A "sbt" value of
* zero is equivalent to a "sbt" value of one tick.
*/
int
pause_sbt(const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags)
{
KASSERT(sbt >= 0, ("pause_sbt: timeout must be >= 0"));
/* silently convert invalid timeouts */
if (sbt == 0)
sbt = tick_sbt;
if ((cold && curthread == &thread0) || kdb_active ||
SCHEDULER_STOPPED()) {
/*
* We delay one second at a time to avoid overflowing the
* system specific DELAY() function(s):
*/
while (sbt >= SBT_1S) {
DELAY(1000000);
sbt -= SBT_1S;
}
/* Do the delay remainder, if any */
sbt = howmany(sbt, SBT_1US);
if (sbt > 0)
DELAY(sbt);
return (EWOULDBLOCK);
}
return (_sleep(&pause_wchan[curcpu], NULL,
(flags & C_CATCH) ? PCATCH : 0, wmesg, sbt, pr, flags));
}
/*
* Potentially release the last reference for refcount. Check for
* unlikely conditions and signal the caller as to whether it was
* the final ref.
*/
bool
refcount_release_last(volatile u_int *count, u_int n, u_int old)
{
u_int waiter;
waiter = old & REFCOUNT_WAITER;
old = REFCOUNT_COUNT(old);
if (__predict_false(n > old || REFCOUNT_SATURATED(old))) {
/*
* Avoid multiple destructor invocations if underflow occurred.
* This is not perfect since the memory backing the containing
* object may already have been reallocated.
*/
_refcount_update_saturated(count);
return (false);
}
/*
* Attempt to atomically clear the waiter bit. Wakeup waiters
* if we are successful.
*/
if (waiter != 0 && atomic_cmpset_int(count, REFCOUNT_WAITER, 0))
wakeup(__DEVOLATILE(u_int *, count));
/*
* Last reference. Signal the user to call the destructor.
*
* Ensure that the destructor sees all updates. The fence_rel
* at the start of refcount_releasen synchronizes with this fence.
*/
atomic_thread_fence_acq();
return (true);
}
/*
* Wait for a refcount wakeup. This does not guarantee that the ref is still
* zero on return and may be subject to transient wakeups. Callers wanting
* a precise answer should use refcount_wait().
*/
void
refcount_sleep(volatile u_int *count, const char *wmesg, int pri)
{
void *wchan;
u_int old;
if (REFCOUNT_COUNT(*count) == 0)
return;
wchan = __DEVOLATILE(void *, count);
sleepq_lock(wchan);
old = *count;
for (;;) {
if (REFCOUNT_COUNT(old) == 0) {
sleepq_release(wchan);
return;
}
if (old & REFCOUNT_WAITER)
break;
if (atomic_fcmpset_int(count, &old, old | REFCOUNT_WAITER))
break;
}
sleepq_add(wchan, NULL, wmesg, 0, 0);
sleepq_wait(wchan, pri);
}
/*
* Make all threads sleeping on the specified identifier runnable.
*/
void
-wakeup(void *ident)
+wakeup(const void *ident)
{
int wakeup_swapper;
sleepq_lock(ident);
wakeup_swapper = sleepq_broadcast(ident, SLEEPQ_SLEEP, 0, 0);
sleepq_release(ident);
if (wakeup_swapper) {
KASSERT(ident != &proc0,
("wakeup and wakeup_swapper and proc0"));
kick_proc0();
}
}
/*
* Make a thread sleeping on the specified identifier runnable.
* May wake more than one thread if a target thread is currently
* swapped out.
*/
void
-wakeup_one(void *ident)
+wakeup_one(const void *ident)
{
int wakeup_swapper;
sleepq_lock(ident);
wakeup_swapper = sleepq_signal(ident, SLEEPQ_SLEEP, 0, 0);
sleepq_release(ident);
if (wakeup_swapper)
kick_proc0();
}
void
-wakeup_any(void *ident)
+wakeup_any(const void *ident)
{
int wakeup_swapper;
sleepq_lock(ident);
wakeup_swapper = sleepq_signal(ident, SLEEPQ_SLEEP | SLEEPQ_UNFAIR,
0, 0);
sleepq_release(ident);
if (wakeup_swapper)
kick_proc0();
}
static void
kdb_switch(void)
{
thread_unlock(curthread);
kdb_backtrace();
kdb_reenter();
panic("%s: did not reenter debugger", __func__);
}
/*
* The machine independent parts of context switching.
*
* The thread lock is required on entry and is no longer held on return.
*/
void
mi_switch(int flags)
{
uint64_t runtime, new_switchtime;
struct thread *td;
td = curthread; /* XXX */
THREAD_LOCK_ASSERT(td, MA_OWNED | MA_NOTRECURSED);
KASSERT(!TD_ON_RUNQ(td), ("mi_switch: called by old code"));
#ifdef INVARIANTS
if (!TD_ON_LOCK(td) && !TD_IS_RUNNING(td))
mtx_assert(&Giant, MA_NOTOWNED);
#endif
KASSERT(td->td_critnest == 1 || panicstr,
("mi_switch: switch in a critical section"));
KASSERT((flags & (SW_INVOL | SW_VOL)) != 0,
("mi_switch: switch must be voluntary or involuntary"));
/*
* Don't perform context switches from the debugger.
*/
if (kdb_active)
kdb_switch();
if (SCHEDULER_STOPPED_TD(td))
return;
if (flags & SW_VOL) {
td->td_ru.ru_nvcsw++;
td->td_swvoltick = ticks;
} else {
td->td_ru.ru_nivcsw++;
td->td_swinvoltick = ticks;
}
#ifdef SCHED_STATS
SCHED_STAT_INC(sched_switch_stats[flags & SW_TYPE_MASK]);
#endif
/*
* Compute the amount of time during which the current
* thread was running, and add that to its total so far.
*/
new_switchtime = cpu_ticks();
runtime = new_switchtime - PCPU_GET(switchtime);
td->td_runtime += runtime;
td->td_incruntime += runtime;
PCPU_SET(switchtime, new_switchtime);
td->td_generation++; /* bump preempt-detect counter */
VM_CNT_INC(v_swtch);
PCPU_SET(switchticks, ticks);
CTR4(KTR_PROC, "mi_switch: old thread %ld (td_sched %p, pid %ld, %s)",
td->td_tid, td_get_sched(td), td->td_proc->p_pid, td->td_name);
#ifdef KDTRACE_HOOKS
if (SDT_PROBES_ENABLED() &&
((flags & SW_PREEMPT) != 0 || ((flags & SW_INVOL) != 0 &&
(flags & SW_TYPE_MASK) == SWT_NEEDRESCHED)))
SDT_PROBE0(sched, , , preempt);
#endif
sched_switch(td, flags);
CTR4(KTR_PROC, "mi_switch: new thread %ld (td_sched %p, pid %ld, %s)",
td->td_tid, td_get_sched(td), td->td_proc->p_pid, td->td_name);
/*
* If the last thread was exiting, finish cleaning it up.
*/
if ((td = PCPU_GET(deadthread))) {
PCPU_SET(deadthread, NULL);
thread_stash(td);
}
spinlock_exit();
}
/*
* Change thread state to be runnable, placing it on the run queue if
* it is in memory. If it is swapped out, return true so our caller
* will know to awaken the swapper.
*
* Requires the thread lock on entry, drops on exit.
*/
int
setrunnable(struct thread *td, int srqflags)
{
int swapin;
THREAD_LOCK_ASSERT(td, MA_OWNED);
KASSERT(td->td_proc->p_state != PRS_ZOMBIE,
("setrunnable: pid %d is a zombie", td->td_proc->p_pid));
swapin = 0;
switch (td->td_state) {
case TDS_RUNNING:
case TDS_RUNQ:
break;
case TDS_CAN_RUN:
KASSERT((td->td_flags & TDF_INMEM) != 0,
("setrunnable: td %p not in mem, flags 0x%X inhibit 0x%X",
td, td->td_flags, td->td_inhibitors));
/* unlocks thread lock according to flags */
sched_wakeup(td, srqflags);
return (0);
case TDS_INHIBITED:
/*
* If we are only inhibited because we are swapped out
* arrange to swap in this process.
*/
if (td->td_inhibitors == TDI_SWAPPED &&
(td->td_flags & TDF_SWAPINREQ) == 0) {
td->td_flags |= TDF_SWAPINREQ;
swapin = 1;
}
break;
default:
panic("setrunnable: state 0x%x", td->td_state);
}
if ((srqflags & (SRQ_HOLD | SRQ_HOLDTD)) == 0)
thread_unlock(td);
return (swapin);
}
/*
* Compute a tenex style load average of a quantity on
* 1, 5 and 15 minute intervals.
*/
static void
loadav(void *arg)
{
int i, nrun;
struct loadavg *avg;
nrun = sched_load();
avg = &averunnable;
for (i = 0; i < 3; i++)
avg->ldavg[i] = (cexp[i] * avg->ldavg[i] +
nrun * FSCALE * (FSCALE - cexp[i])) >> FSHIFT;
/*
* Schedule the next update to occur after 5 seconds, but add a
* random variation to avoid synchronisation with processes that
* run at regular intervals.
*/
callout_reset_sbt(&loadav_callout,
SBT_1US * (4000000 + (int)(random() % 2000001)), SBT_1US,
loadav, NULL, C_DIRECT_EXEC | C_PREL(32));
}
/* ARGSUSED */
static void
synch_setup(void *dummy)
{
callout_init(&loadav_callout, 1);
/* Kick off timeout driven events by calling first time. */
loadav(NULL);
}
int
should_yield(void)
{
return ((u_int)ticks - (u_int)curthread->td_swvoltick >= hogticks);
}
void
maybe_yield(void)
{
if (should_yield())
kern_yield(PRI_USER);
}
void
kern_yield(int prio)
{
struct thread *td;
td = curthread;
DROP_GIANT();
thread_lock(td);
if (prio == PRI_USER)
prio = td->td_user_pri;
if (prio >= 0)
sched_prio(td, prio);
mi_switch(SW_VOL | SWT_RELINQUISH);
PICKUP_GIANT();
}
/*
* General purpose yield system call.
*/
int
sys_yield(struct thread *td, struct yield_args *uap)
{
thread_lock(td);
if (PRI_BASE(td->td_pri_class) == PRI_TIMESHARE)
sched_prio(td, PRI_MAX_TIMESHARE);
mi_switch(SW_VOL | SWT_RELINQUISH);
td->td_retval[0] = 0;
return (0);
}
Index: head/sys/kern/subr_sleepqueue.c
===================================================================
--- head/sys/kern/subr_sleepqueue.c (revision 356056)
+++ head/sys/kern/subr_sleepqueue.c (revision 356057)
@@ -1,1504 +1,1504 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2004 John Baldwin <jhb@FreeBSD.org>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* Implementation of sleep queues used to hold queue of threads blocked on
* a wait channel. Sleep queues are different from turnstiles in that wait
* channels are not owned by anyone, so there is no priority propagation.
* Sleep queues can also provide a timeout and can also be interrupted by
* signals. That said, there are several similarities between the turnstile
* and sleep queue implementations. (Note: turnstiles were implemented
* first.) For example, both use a hash table of the same size where each
* bucket is referred to as a "chain" that contains both a spin lock and
* a linked list of queues. An individual queue is located by using a hash
* to pick a chain, locking the chain, and then walking the chain searching
* for the queue. This means that a wait channel object does not need to
* embed its queue head just as locks do not embed their turnstile queue
* head. Threads also carry around a sleep queue that they lend to the
* wait channel when blocking. Just as in turnstiles, the queue includes
* a free list of the sleep queues of other threads blocked on the same
* wait channel in the case of multiple waiters.
*
* Some additional functionality provided by sleep queues include the
* ability to set a timeout. The timeout is managed using a per-thread
* callout that resumes a thread if it is asleep. A thread may also
* catch signals while it is asleep (aka an interruptible sleep). The
* signal code uses sleepq_abort() to interrupt a sleeping thread. Finally,
* sleep queues also provide some extra assertions. One is not allowed to
* mix the sleep/wakeup and cv APIs for a given wait channel. Also, one
* must consistently use the same lock to synchronize with a wait channel,
* though this check is currently only a warning for sleep/wakeup due to
* pre-existing abuse of that API. The same lock must also be held when
* awakening threads, though that is currently only enforced for condition
* variables.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_sleepqueue_profiling.h"
#include "opt_ddb.h"
#include "opt_sched.h"
#include "opt_stack.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/lock.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/sbuf.h>
#include <sys/sched.h>
#include <sys/sdt.h>
#include <sys/signalvar.h>
#include <sys/sleepqueue.h>
#include <sys/stack.h>
#include <sys/sysctl.h>
#include <sys/time.h>
#ifdef EPOCH_TRACE
#include <sys/epoch.h>
#endif
#include <machine/atomic.h>
#include <vm/uma.h>
#ifdef DDB
#include <ddb/ddb.h>
#endif
/*
* Constants for the hash table of sleep queue chains.
* SC_TABLESIZE must be a power of two for SC_MASK to work properly.
*/
#ifndef SC_TABLESIZE
#define SC_TABLESIZE 256
#endif
CTASSERT(powerof2(SC_TABLESIZE));
#define SC_MASK (SC_TABLESIZE - 1)
#define SC_SHIFT 8
#define SC_HASH(wc) ((((uintptr_t)(wc) >> SC_SHIFT) ^ (uintptr_t)(wc)) & \
SC_MASK)
#define SC_LOOKUP(wc) &sleepq_chains[SC_HASH(wc)]
#define NR_SLEEPQS 2
/*
* There are two different lists of sleep queues. Both lists are connected
* via the sq_hash entries. The first list is the sleep queue chain list
* that a sleep queue is on when it is attached to a wait channel. The
* second list is the free list hung off of a sleep queue that is attached
* to a wait channel.
*
* Each sleep queue also contains the wait channel it is attached to, the
* list of threads blocked on that wait channel, flags specific to the
* wait channel, and the lock used to synchronize with a wait channel.
* The flags are used to catch mismatches between the various consumers
* of the sleep queue API (e.g. sleep/wakeup and condition variables).
* The lock pointer is only used when invariants are enabled for various
* debugging checks.
*
* Locking key:
* c - sleep queue chain lock
*/
struct sleepqueue {
struct threadqueue sq_blocked[NR_SLEEPQS]; /* (c) Blocked threads. */
u_int sq_blockedcnt[NR_SLEEPQS]; /* (c) N. of blocked threads. */
LIST_ENTRY(sleepqueue) sq_hash; /* (c) Chain and free list. */
LIST_HEAD(, sleepqueue) sq_free; /* (c) Free queues. */
- void *sq_wchan; /* (c) Wait channel. */
+ const void *sq_wchan; /* (c) Wait channel. */
int sq_type; /* (c) Queue type. */
#ifdef INVARIANTS
struct lock_object *sq_lock; /* (c) Associated lock. */
#endif
};
struct sleepqueue_chain {
LIST_HEAD(, sleepqueue) sc_queues; /* List of sleep queues. */
struct mtx sc_lock; /* Spin lock for this chain. */
#ifdef SLEEPQUEUE_PROFILING
u_int sc_depth; /* Length of sc_queues. */
u_int sc_max_depth; /* Max length of sc_queues. */
#endif
} __aligned(CACHE_LINE_SIZE);
#ifdef SLEEPQUEUE_PROFILING
u_int sleepq_max_depth;
static SYSCTL_NODE(_debug, OID_AUTO, sleepq, CTLFLAG_RD, 0, "sleepq profiling");
static SYSCTL_NODE(_debug_sleepq, OID_AUTO, chains, CTLFLAG_RD, 0,
"sleepq chain stats");
SYSCTL_UINT(_debug_sleepq, OID_AUTO, max_depth, CTLFLAG_RD, &sleepq_max_depth,
0, "maxmimum depth achieved of a single chain");
static void sleepq_profile(const char *wmesg);
static int prof_enabled;
#endif
static struct sleepqueue_chain sleepq_chains[SC_TABLESIZE];
static uma_zone_t sleepq_zone;
/*
* Prototypes for non-exported routines.
*/
-static int sleepq_catch_signals(void *wchan, int pri);
+static int sleepq_catch_signals(const void *wchan, int pri);
static inline int sleepq_check_signals(void);
static inline int sleepq_check_timeout(void);
#ifdef INVARIANTS
static void sleepq_dtor(void *mem, int size, void *arg);
#endif
static int sleepq_init(void *mem, int size, int flags);
static int sleepq_resume_thread(struct sleepqueue *sq, struct thread *td,
int pri, int srqflags);
static void sleepq_remove_thread(struct sleepqueue *sq, struct thread *td);
-static void sleepq_switch(void *wchan, int pri);
+static void sleepq_switch(const void *wchan, int pri);
static void sleepq_timeout(void *arg);
SDT_PROBE_DECLARE(sched, , , sleep);
SDT_PROBE_DECLARE(sched, , , wakeup);
/*
* Initialize SLEEPQUEUE_PROFILING specific sysctl nodes.
* Note that it must happen after sleepinit() has been fully executed, so
* it must happen after SI_SUB_KMEM SYSINIT() subsystem setup.
*/
#ifdef SLEEPQUEUE_PROFILING
static void
init_sleepqueue_profiling(void)
{
char chain_name[10];
struct sysctl_oid *chain_oid;
u_int i;
for (i = 0; i < SC_TABLESIZE; i++) {
snprintf(chain_name, sizeof(chain_name), "%u", i);
chain_oid = SYSCTL_ADD_NODE(NULL,
SYSCTL_STATIC_CHILDREN(_debug_sleepq_chains), OID_AUTO,
chain_name, CTLFLAG_RD, NULL, "sleepq chain stats");
SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO,
"depth", CTLFLAG_RD, &sleepq_chains[i].sc_depth, 0, NULL);
SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO,
"max_depth", CTLFLAG_RD, &sleepq_chains[i].sc_max_depth, 0,
NULL);
}
}
SYSINIT(sleepqueue_profiling, SI_SUB_LOCK, SI_ORDER_ANY,
init_sleepqueue_profiling, NULL);
#endif
/*
* Early initialization of sleep queues that is called from the sleepinit()
* SYSINIT.
*/
void
init_sleepqueues(void)
{
int i;
for (i = 0; i < SC_TABLESIZE; i++) {
LIST_INIT(&sleepq_chains[i].sc_queues);
mtx_init(&sleepq_chains[i].sc_lock, "sleepq chain", NULL,
MTX_SPIN);
}
sleepq_zone = uma_zcreate("SLEEPQUEUE", sizeof(struct sleepqueue),
#ifdef INVARIANTS
NULL, sleepq_dtor, sleepq_init, NULL, UMA_ALIGN_CACHE, 0);
#else
NULL, NULL, sleepq_init, NULL, UMA_ALIGN_CACHE, 0);
#endif
thread0.td_sleepqueue = sleepq_alloc();
}
/*
* Get a sleep queue for a new thread.
*/
struct sleepqueue *
sleepq_alloc(void)
{
return (uma_zalloc(sleepq_zone, M_WAITOK));
}
/*
* Free a sleep queue when a thread is destroyed.
*/
void
sleepq_free(struct sleepqueue *sq)
{
uma_zfree(sleepq_zone, sq);
}
/*
* Lock the sleep queue chain associated with the specified wait channel.
*/
void
-sleepq_lock(void *wchan)
+sleepq_lock(const void *wchan)
{
struct sleepqueue_chain *sc;
sc = SC_LOOKUP(wchan);
mtx_lock_spin(&sc->sc_lock);
}
/*
* Look up the sleep queue associated with a given wait channel in the hash
* table locking the associated sleep queue chain. If no queue is found in
* the table, NULL is returned.
*/
struct sleepqueue *
-sleepq_lookup(void *wchan)
+sleepq_lookup(const void *wchan)
{
struct sleepqueue_chain *sc;
struct sleepqueue *sq;
KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
sc = SC_LOOKUP(wchan);
mtx_assert(&sc->sc_lock, MA_OWNED);
LIST_FOREACH(sq, &sc->sc_queues, sq_hash)
if (sq->sq_wchan == wchan)
return (sq);
return (NULL);
}
/*
* Unlock the sleep queue chain associated with a given wait channel.
*/
void
-sleepq_release(void *wchan)
+sleepq_release(const void *wchan)
{
struct sleepqueue_chain *sc;
sc = SC_LOOKUP(wchan);
mtx_unlock_spin(&sc->sc_lock);
}
/*
* Places the current thread on the sleep queue for the specified wait
* channel. If INVARIANTS is enabled, then it associates the passed in
* lock with the sleepq to make sure it is held when that sleep queue is
* woken up.
*/
void
-sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg, int flags,
- int queue)
+sleepq_add(const void *wchan, struct lock_object *lock, const char *wmesg,
+ int flags, int queue)
{
struct sleepqueue_chain *sc;
struct sleepqueue *sq;
struct thread *td;
td = curthread;
sc = SC_LOOKUP(wchan);
mtx_assert(&sc->sc_lock, MA_OWNED);
MPASS(td->td_sleepqueue != NULL);
MPASS(wchan != NULL);
MPASS((queue >= 0) && (queue < NR_SLEEPQS));
/* If this thread is not allowed to sleep, die a horrible death. */
if (__predict_false(!THREAD_CAN_SLEEP())) {
#ifdef EPOCH_TRACE
epoch_trace_list(curthread);
#endif
KASSERT(1,
("%s: td %p to sleep on wchan %p with sleeping prohibited",
__func__, td, wchan));
}
/* Look up the sleep queue associated with the wait channel 'wchan'. */
sq = sleepq_lookup(wchan);
/*
* If the wait channel does not already have a sleep queue, use
* this thread's sleep queue. Otherwise, insert the current thread
* into the sleep queue already in use by this wait channel.
*/
if (sq == NULL) {
#ifdef INVARIANTS
int i;
sq = td->td_sleepqueue;
for (i = 0; i < NR_SLEEPQS; i++) {
KASSERT(TAILQ_EMPTY(&sq->sq_blocked[i]),
("thread's sleep queue %d is not empty", i));
KASSERT(sq->sq_blockedcnt[i] == 0,
("thread's sleep queue %d count mismatches", i));
}
KASSERT(LIST_EMPTY(&sq->sq_free),
("thread's sleep queue has a non-empty free list"));
KASSERT(sq->sq_wchan == NULL, ("stale sq_wchan pointer"));
sq->sq_lock = lock;
#endif
#ifdef SLEEPQUEUE_PROFILING
sc->sc_depth++;
if (sc->sc_depth > sc->sc_max_depth) {
sc->sc_max_depth = sc->sc_depth;
if (sc->sc_max_depth > sleepq_max_depth)
sleepq_max_depth = sc->sc_max_depth;
}
#endif
sq = td->td_sleepqueue;
LIST_INSERT_HEAD(&sc->sc_queues, sq, sq_hash);
sq->sq_wchan = wchan;
sq->sq_type = flags & SLEEPQ_TYPE;
} else {
MPASS(wchan == sq->sq_wchan);
MPASS(lock == sq->sq_lock);
MPASS((flags & SLEEPQ_TYPE) == sq->sq_type);
LIST_INSERT_HEAD(&sq->sq_free, td->td_sleepqueue, sq_hash);
}
thread_lock(td);
TAILQ_INSERT_TAIL(&sq->sq_blocked[queue], td, td_slpq);
sq->sq_blockedcnt[queue]++;
td->td_sleepqueue = NULL;
td->td_sqqueue = queue;
td->td_wchan = wchan;
td->td_wmesg = wmesg;
if (flags & SLEEPQ_INTERRUPTIBLE) {
td->td_intrval = 0;
td->td_flags |= TDF_SINTR;
}
td->td_flags &= ~TDF_TIMEOUT;
thread_unlock(td);
}
/*
* Sets a timeout that will remove the current thread from the specified
* sleep queue after timo ticks if the thread has not already been awakened.
*/
void
-sleepq_set_timeout_sbt(void *wchan, sbintime_t sbt, sbintime_t pr,
+sleepq_set_timeout_sbt(const void *wchan, sbintime_t sbt, sbintime_t pr,
int flags)
{
struct sleepqueue_chain *sc __unused;
struct thread *td;
sbintime_t pr1;
td = curthread;
sc = SC_LOOKUP(wchan);
mtx_assert(&sc->sc_lock, MA_OWNED);
MPASS(TD_ON_SLEEPQ(td));
MPASS(td->td_sleepqueue == NULL);
MPASS(wchan != NULL);
if (cold && td == &thread0)
panic("timed sleep before timers are working");
KASSERT(td->td_sleeptimo == 0, ("td %d %p td_sleeptimo %jx",
td->td_tid, td, (uintmax_t)td->td_sleeptimo));
thread_lock(td);
callout_when(sbt, pr, flags, &td->td_sleeptimo, &pr1);
thread_unlock(td);
callout_reset_sbt_on(&td->td_slpcallout, td->td_sleeptimo, pr1,
sleepq_timeout, td, PCPU_GET(cpuid), flags | C_PRECALC |
C_DIRECT_EXEC);
}
/*
* Return the number of actual sleepers for the specified queue.
*/
u_int
-sleepq_sleepcnt(void *wchan, int queue)
+sleepq_sleepcnt(const void *wchan, int queue)
{
struct sleepqueue *sq;
KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
MPASS((queue >= 0) && (queue < NR_SLEEPQS));
sq = sleepq_lookup(wchan);
if (sq == NULL)
return (0);
return (sq->sq_blockedcnt[queue]);
}
/*
* Marks the pending sleep of the current thread as interruptible and
* makes an initial check for pending signals before putting a thread
* to sleep. Enters and exits with the thread lock held. Thread lock
* may have transitioned from the sleepq lock to a run lock.
*/
static int
-sleepq_catch_signals(void *wchan, int pri)
+sleepq_catch_signals(const void *wchan, int pri)
{
struct sleepqueue_chain *sc;
struct sleepqueue *sq;
struct thread *td;
struct proc *p;
struct sigacts *ps;
int sig, ret;
ret = 0;
td = curthread;
p = curproc;
sc = SC_LOOKUP(wchan);
mtx_assert(&sc->sc_lock, MA_OWNED);
MPASS(wchan != NULL);
if ((td->td_pflags & TDP_WAKEUP) != 0) {
td->td_pflags &= ~TDP_WAKEUP;
ret = EINTR;
thread_lock(td);
goto out;
}
/*
* See if there are any pending signals or suspension requests for this
* thread. If not, we can switch immediately.
*/
thread_lock(td);
if ((td->td_flags & (TDF_NEEDSIGCHK | TDF_NEEDSUSPCHK)) != 0) {
thread_unlock(td);
mtx_unlock_spin(&sc->sc_lock);
CTR3(KTR_PROC, "sleepq catching signals: thread %p (pid %ld, %s)",
(void *)td, (long)p->p_pid, td->td_name);
PROC_LOCK(p);
/*
* Check for suspension first. Checking for signals and then
* suspending could result in a missed signal, since a signal
* can be delivered while this thread is suspended.
*/
if ((td->td_flags & TDF_NEEDSUSPCHK) != 0) {
ret = thread_suspend_check(1);
MPASS(ret == 0 || ret == EINTR || ret == ERESTART);
if (ret != 0) {
PROC_UNLOCK(p);
mtx_lock_spin(&sc->sc_lock);
thread_lock(td);
goto out;
}
}
if ((td->td_flags & TDF_NEEDSIGCHK) != 0) {
ps = p->p_sigacts;
mtx_lock(&ps->ps_mtx);
sig = cursig(td);
if (sig == -1) {
mtx_unlock(&ps->ps_mtx);
KASSERT((td->td_flags & TDF_SBDRY) != 0,
("lost TDF_SBDRY"));
KASSERT(TD_SBDRY_INTR(td),
("lost TDF_SERESTART of TDF_SEINTR"));
KASSERT((td->td_flags &
(TDF_SEINTR | TDF_SERESTART)) !=
(TDF_SEINTR | TDF_SERESTART),
("both TDF_SEINTR and TDF_SERESTART"));
ret = TD_SBDRY_ERRNO(td);
} else if (sig != 0) {
ret = SIGISMEMBER(ps->ps_sigintr, sig) ?
EINTR : ERESTART;
mtx_unlock(&ps->ps_mtx);
} else {
mtx_unlock(&ps->ps_mtx);
}
/*
* Do not go into sleep if this thread was the
* ptrace(2) attach leader. cursig() consumed
* SIGSTOP from PT_ATTACH, but we usually act
* on the signal by interrupting sleep, and
* should do that here as well.
*/
if ((td->td_dbgflags & TDB_FSTP) != 0) {
if (ret == 0)
ret = EINTR;
td->td_dbgflags &= ~TDB_FSTP;
}
}
/*
* Lock the per-process spinlock prior to dropping the PROC_LOCK
* to avoid a signal delivery race. PROC_LOCK, PROC_SLOCK, and
* thread_lock() are currently held in tdsendsignal().
*/
PROC_SLOCK(p);
mtx_lock_spin(&sc->sc_lock);
PROC_UNLOCK(p);
thread_lock(td);
PROC_SUNLOCK(p);
}
if (ret == 0) {
sleepq_switch(wchan, pri);
return (0);
}
out:
/*
* There were pending signals and this thread is still
* on the sleep queue, remove it from the sleep queue.
*/
if (TD_ON_SLEEPQ(td)) {
sq = sleepq_lookup(wchan);
sleepq_remove_thread(sq, td);
}
MPASS(td->td_lock != &sc->sc_lock);
mtx_unlock_spin(&sc->sc_lock);
thread_unlock(td);
return (ret);
}
/*
* Switches to another thread if we are still asleep on a sleep queue.
* Returns with thread lock.
*/
static void
-sleepq_switch(void *wchan, int pri)
+sleepq_switch(const void *wchan, int pri)
{
struct sleepqueue_chain *sc;
struct sleepqueue *sq;
struct thread *td;
bool rtc_changed;
td = curthread;
sc = SC_LOOKUP(wchan);
mtx_assert(&sc->sc_lock, MA_OWNED);
THREAD_LOCK_ASSERT(td, MA_OWNED);
/*
* If we have a sleep queue, then we've already been woken up, so
* just return.
*/
if (td->td_sleepqueue != NULL) {
mtx_unlock_spin(&sc->sc_lock);
thread_unlock(td);
return;
}
/*
* If TDF_TIMEOUT is set, then our sleep has been timed out
* already but we are still on the sleep queue, so dequeue the
* thread and return.
*
* Do the same if the real-time clock has been adjusted since this
* thread calculated its timeout based on that clock. This handles
* the following race:
* - The Ts thread needs to sleep until an absolute real-clock time.
* It copies the global rtc_generation into curthread->td_rtcgen,
* reads the RTC, and calculates a sleep duration based on that time.
* See umtxq_sleep() for an example.
* - The Tc thread adjusts the RTC, bumps rtc_generation, and wakes
* threads that are sleeping until an absolute real-clock time.
* See tc_setclock() and the POSIX specification of clock_settime().
* - Ts reaches the code below. It holds the sleepqueue chain lock,
* so Tc has finished waking, so this thread must test td_rtcgen.
* (The declaration of td_rtcgen refers to this comment.)
*/
rtc_changed = td->td_rtcgen != 0 && td->td_rtcgen != rtc_generation;
if ((td->td_flags & TDF_TIMEOUT) || rtc_changed) {
if (rtc_changed) {
td->td_rtcgen = 0;
}
MPASS(TD_ON_SLEEPQ(td));
sq = sleepq_lookup(wchan);
sleepq_remove_thread(sq, td);
mtx_unlock_spin(&sc->sc_lock);
thread_unlock(td);
return;
}
#ifdef SLEEPQUEUE_PROFILING
if (prof_enabled)
sleepq_profile(td->td_wmesg);
#endif
MPASS(td->td_sleepqueue == NULL);
sched_sleep(td, pri);
thread_lock_set(td, &sc->sc_lock);
SDT_PROBE0(sched, , , sleep);
TD_SET_SLEEPING(td);
mi_switch(SW_VOL | SWT_SLEEPQ);
KASSERT(TD_IS_RUNNING(td), ("running but not TDS_RUNNING"));
CTR3(KTR_PROC, "sleepq resume: thread %p (pid %ld, %s)",
(void *)td, (long)td->td_proc->p_pid, (void *)td->td_name);
}
/*
* Check to see if we timed out.
*/
static inline int
sleepq_check_timeout(void)
{
struct thread *td;
int res;
res = 0;
td = curthread;
if (td->td_sleeptimo != 0) {
if (td->td_sleeptimo <= sbinuptime())
res = EWOULDBLOCK;
td->td_sleeptimo = 0;
}
return (res);
}
/*
* Check to see if we were awoken by a signal.
*/
static inline int
sleepq_check_signals(void)
{
struct thread *td;
td = curthread;
KASSERT((td->td_flags & TDF_SINTR) == 0,
("thread %p still in interruptible sleep?", td));
return (td->td_intrval);
}
/*
* Block the current thread until it is awakened from its sleep queue.
*/
void
-sleepq_wait(void *wchan, int pri)
+sleepq_wait(const void *wchan, int pri)
{
struct thread *td;
td = curthread;
MPASS(!(td->td_flags & TDF_SINTR));
thread_lock(td);
sleepq_switch(wchan, pri);
}
/*
* Block the current thread until it is awakened from its sleep queue
* or it is interrupted by a signal.
*/
int
-sleepq_wait_sig(void *wchan, int pri)
+sleepq_wait_sig(const void *wchan, int pri)
{
int rcatch;
rcatch = sleepq_catch_signals(wchan, pri);
if (rcatch)
return (rcatch);
return (sleepq_check_signals());
}
/*
* Block the current thread until it is awakened from its sleep queue
* or it times out while waiting.
*/
int
-sleepq_timedwait(void *wchan, int pri)
+sleepq_timedwait(const void *wchan, int pri)
{
struct thread *td;
td = curthread;
MPASS(!(td->td_flags & TDF_SINTR));
thread_lock(td);
sleepq_switch(wchan, pri);
return (sleepq_check_timeout());
}
/*
* Block the current thread until it is awakened from its sleep queue,
* it is interrupted by a signal, or it times out waiting to be awakened.
*/
int
-sleepq_timedwait_sig(void *wchan, int pri)
+sleepq_timedwait_sig(const void *wchan, int pri)
{
int rcatch, rvalt, rvals;
rcatch = sleepq_catch_signals(wchan, pri);
/* We must always call check_timeout() to clear sleeptimo. */
rvalt = sleepq_check_timeout();
rvals = sleepq_check_signals();
if (rcatch)
return (rcatch);
if (rvals)
return (rvals);
return (rvalt);
}
/*
* Returns the type of sleepqueue given a waitchannel.
*/
int
-sleepq_type(void *wchan)
+sleepq_type(const void *wchan)
{
struct sleepqueue *sq;
int type;
MPASS(wchan != NULL);
sq = sleepq_lookup(wchan);
if (sq == NULL)
return (-1);
type = sq->sq_type;
return (type);
}
/*
* Removes a thread from a sleep queue and makes it
* runnable.
*
* Requires the sc chain locked on entry. If SRQ_HOLD is specified it will
* be locked on return. Returns without the thread lock held.
*/
static int
sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, int pri,
int srqflags)
{
struct sleepqueue_chain *sc;
bool drop;
MPASS(td != NULL);
MPASS(sq->sq_wchan != NULL);
MPASS(td->td_wchan == sq->sq_wchan);
sc = SC_LOOKUP(sq->sq_wchan);
mtx_assert(&sc->sc_lock, MA_OWNED);
/*
* Avoid recursing on the chain lock. If the locks don't match we
* need to acquire the thread lock which setrunnable will drop for
* us. In this case we need to drop the chain lock afterwards.
*
* There is no race that will make td_lock equal to sc_lock because
* we hold sc_lock.
*/
drop = false;
if (!TD_IS_SLEEPING(td)) {
thread_lock(td);
drop = true;
} else
thread_lock_block_wait(td);
/* Remove thread from the sleepq. */
sleepq_remove_thread(sq, td);
/* If we're done with the sleepqueue release it. */
if ((srqflags & SRQ_HOLD) == 0 && drop)
mtx_unlock_spin(&sc->sc_lock);
/* Adjust priority if requested. */
MPASS(pri == 0 || (pri >= PRI_MIN && pri <= PRI_MAX));
if (pri != 0 && td->td_priority > pri &&
PRI_BASE(td->td_pri_class) == PRI_TIMESHARE)
sched_prio(td, pri);
/*
* Note that thread td might not be sleeping if it is running
* sleepq_catch_signals() on another CPU or is blocked on its
* proc lock to check signals. There's no need to mark the
* thread runnable in that case.
*/
if (TD_IS_SLEEPING(td)) {
MPASS(!drop);
TD_CLR_SLEEPING(td);
return (setrunnable(td, srqflags));
}
MPASS(drop);
thread_unlock(td);
return (0);
}
static void
sleepq_remove_thread(struct sleepqueue *sq, struct thread *td)
{
struct sleepqueue_chain *sc __unused;
MPASS(td != NULL);
MPASS(sq->sq_wchan != NULL);
MPASS(td->td_wchan == sq->sq_wchan);
MPASS(td->td_sqqueue < NR_SLEEPQS && td->td_sqqueue >= 0);
THREAD_LOCK_ASSERT(td, MA_OWNED);
sc = SC_LOOKUP(sq->sq_wchan);
mtx_assert(&sc->sc_lock, MA_OWNED);
SDT_PROBE2(sched, , , wakeup, td, td->td_proc);
/* Remove the thread from the queue. */
sq->sq_blockedcnt[td->td_sqqueue]--;
TAILQ_REMOVE(&sq->sq_blocked[td->td_sqqueue], td, td_slpq);
/*
* Get a sleep queue for this thread. If this is the last waiter,
* use the queue itself and take it out of the chain, otherwise,
* remove a queue from the free list.
*/
if (LIST_EMPTY(&sq->sq_free)) {
td->td_sleepqueue = sq;
#ifdef INVARIANTS
sq->sq_wchan = NULL;
#endif
#ifdef SLEEPQUEUE_PROFILING
sc->sc_depth--;
#endif
} else
td->td_sleepqueue = LIST_FIRST(&sq->sq_free);
LIST_REMOVE(td->td_sleepqueue, sq_hash);
if ((td->td_flags & TDF_TIMEOUT) == 0 && td->td_sleeptimo != 0)
/*
* We ignore the situation where timeout subsystem was
* unable to stop our callout. The struct thread is
* type-stable, the callout will use the correct
* memory when running. The checks of the
* td_sleeptimo value in this function and in
* sleepq_timeout() ensure that the thread does not
* get spurious wakeups, even if the callout was reset
* or thread reused.
*/
callout_stop(&td->td_slpcallout);
td->td_wmesg = NULL;
td->td_wchan = NULL;
td->td_flags &= ~(TDF_SINTR | TDF_TIMEOUT);
CTR3(KTR_PROC, "sleepq_wakeup: thread %p (pid %ld, %s)",
(void *)td, (long)td->td_proc->p_pid, td->td_name);
}
#ifdef INVARIANTS
/*
* UMA zone item deallocator.
*/
static void
sleepq_dtor(void *mem, int size, void *arg)
{
struct sleepqueue *sq;
int i;
sq = mem;
for (i = 0; i < NR_SLEEPQS; i++) {
MPASS(TAILQ_EMPTY(&sq->sq_blocked[i]));
MPASS(sq->sq_blockedcnt[i] == 0);
}
}
#endif
/*
* UMA zone item initializer.
*/
static int
sleepq_init(void *mem, int size, int flags)
{
struct sleepqueue *sq;
int i;
bzero(mem, size);
sq = mem;
for (i = 0; i < NR_SLEEPQS; i++) {
TAILQ_INIT(&sq->sq_blocked[i]);
sq->sq_blockedcnt[i] = 0;
}
LIST_INIT(&sq->sq_free);
return (0);
}
/*
* Find thread sleeping on a wait channel and resume it.
*/
int
-sleepq_signal(void *wchan, int flags, int pri, int queue)
+sleepq_signal(const void *wchan, int flags, int pri, int queue)
{
struct sleepqueue_chain *sc;
struct sleepqueue *sq;
struct threadqueue *head;
struct thread *td, *besttd;
int wakeup_swapper;
CTR2(KTR_PROC, "sleepq_signal(%p, %d)", wchan, flags);
KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
MPASS((queue >= 0) && (queue < NR_SLEEPQS));
sq = sleepq_lookup(wchan);
if (sq == NULL)
return (0);
KASSERT(sq->sq_type == (flags & SLEEPQ_TYPE),
("%s: mismatch between sleep/wakeup and cv_*", __func__));
head = &sq->sq_blocked[queue];
if (flags & SLEEPQ_UNFAIR) {
/*
* Find the most recently sleeping thread, but try to
* skip threads still in process of context switch to
* avoid spinning on the thread lock.
*/
sc = SC_LOOKUP(wchan);
besttd = TAILQ_LAST_FAST(head, thread, td_slpq);
while (besttd->td_lock != &sc->sc_lock) {
td = TAILQ_PREV_FAST(besttd, head, thread, td_slpq);
if (td == NULL)
break;
besttd = td;
}
} else {
/*
* Find the highest priority thread on the queue. If there
* is a tie, use the thread that first appears in the queue
* as it has been sleeping the longest since threads are
* always added to the tail of sleep queues.
*/
besttd = td = TAILQ_FIRST(head);
while ((td = TAILQ_NEXT(td, td_slpq)) != NULL) {
if (td->td_priority < besttd->td_priority)
besttd = td;
}
}
MPASS(besttd != NULL);
wakeup_swapper = sleepq_resume_thread(sq, besttd, pri, SRQ_HOLD);
return (wakeup_swapper);
}
static bool
match_any(struct thread *td __unused)
{
return (true);
}
/*
* Resume all threads sleeping on a specified wait channel.
*/
int
-sleepq_broadcast(void *wchan, int flags, int pri, int queue)
+sleepq_broadcast(const void *wchan, int flags, int pri, int queue)
{
struct sleepqueue *sq;
CTR2(KTR_PROC, "sleepq_broadcast(%p, %d)", wchan, flags);
KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
MPASS((queue >= 0) && (queue < NR_SLEEPQS));
sq = sleepq_lookup(wchan);
if (sq == NULL)
return (0);
KASSERT(sq->sq_type == (flags & SLEEPQ_TYPE),
("%s: mismatch between sleep/wakeup and cv_*", __func__));
return (sleepq_remove_matching(sq, queue, match_any, pri));
}
/*
* Resume threads on the sleep queue that match the given predicate.
*/
int
sleepq_remove_matching(struct sleepqueue *sq, int queue,
bool (*matches)(struct thread *), int pri)
{
struct thread *td, *tdn;
int wakeup_swapper;
/*
* The last thread will be given ownership of sq and may
* re-enqueue itself before sleepq_resume_thread() returns,
* so we must cache the "next" queue item at the beginning
* of the final iteration.
*/
wakeup_swapper = 0;
TAILQ_FOREACH_SAFE(td, &sq->sq_blocked[queue], td_slpq, tdn) {
if (matches(td))
wakeup_swapper |= sleepq_resume_thread(sq, td, pri,
SRQ_HOLD);
}
return (wakeup_swapper);
}
/*
* Time sleeping threads out. When the timeout expires, the thread is
* removed from the sleep queue and made runnable if it is still asleep.
*/
static void
sleepq_timeout(void *arg)
{
struct sleepqueue_chain *sc __unused;
struct sleepqueue *sq;
struct thread *td;
- void *wchan;
+ const void *wchan;
int wakeup_swapper;
td = arg;
CTR3(KTR_PROC, "sleepq_timeout: thread %p (pid %ld, %s)",
(void *)td, (long)td->td_proc->p_pid, (void *)td->td_name);
thread_lock(td);
if (td->td_sleeptimo == 0 || td->td_sleeptimo > sbinuptime()) {
/*
* The thread does not want a timeout (yet).
*/
} else if (TD_IS_SLEEPING(td) && TD_ON_SLEEPQ(td)) {
/*
* See if the thread is asleep and get the wait
* channel if it is.
*/
wchan = td->td_wchan;
sc = SC_LOOKUP(wchan);
THREAD_LOCKPTR_ASSERT(td, &sc->sc_lock);
sq = sleepq_lookup(wchan);
MPASS(sq != NULL);
td->td_flags |= TDF_TIMEOUT;
wakeup_swapper = sleepq_resume_thread(sq, td, 0, 0);
if (wakeup_swapper)
kick_proc0();
return;
} else if (TD_ON_SLEEPQ(td)) {
/*
* If the thread is on the SLEEPQ but isn't sleeping
* yet, it can either be on another CPU in between
* sleepq_add() and one of the sleepq_*wait*()
* routines or it can be in sleepq_catch_signals().
*/
td->td_flags |= TDF_TIMEOUT;
}
thread_unlock(td);
}
/*
* Resumes a specific thread from the sleep queue associated with a specific
* wait channel if it is on that queue.
*/
void
-sleepq_remove(struct thread *td, void *wchan)
+sleepq_remove(struct thread *td, const void *wchan)
{
struct sleepqueue_chain *sc;
struct sleepqueue *sq;
int wakeup_swapper;
/*
* Look up the sleep queue for this wait channel, then re-check
* that the thread is asleep on that channel, if it is not, then
* bail.
*/
MPASS(wchan != NULL);
sc = SC_LOOKUP(wchan);
mtx_lock_spin(&sc->sc_lock);
/*
* We can not lock the thread here as it may be sleeping on a
* different sleepq. However, holding the sleepq lock for this
* wchan can guarantee that we do not miss a wakeup for this
* channel. The asserts below will catch any false positives.
*/
if (!TD_ON_SLEEPQ(td) || td->td_wchan != wchan) {
mtx_unlock_spin(&sc->sc_lock);
return;
}
/* Thread is asleep on sleep queue sq, so wake it up. */
sq = sleepq_lookup(wchan);
MPASS(sq != NULL);
MPASS(td->td_wchan == wchan);
wakeup_swapper = sleepq_resume_thread(sq, td, 0, 0);
if (wakeup_swapper)
kick_proc0();
}
/*
* Abort a thread as if an interrupt had occurred. Only abort
* interruptible waits (unfortunately it isn't safe to abort others).
*
* Requires thread lock on entry, releases on return.
*/
int
sleepq_abort(struct thread *td, int intrval)
{
struct sleepqueue *sq;
- void *wchan;
+ const void *wchan;
THREAD_LOCK_ASSERT(td, MA_OWNED);
MPASS(TD_ON_SLEEPQ(td));
MPASS(td->td_flags & TDF_SINTR);
MPASS(intrval == EINTR || intrval == ERESTART);
/*
* If the TDF_TIMEOUT flag is set, just leave. A
* timeout is scheduled anyhow.
*/
if (td->td_flags & TDF_TIMEOUT) {
thread_unlock(td);
return (0);
}
CTR3(KTR_PROC, "sleepq_abort: thread %p (pid %ld, %s)",
(void *)td, (long)td->td_proc->p_pid, (void *)td->td_name);
td->td_intrval = intrval;
/*
* If the thread has not slept yet it will find the signal in
* sleepq_catch_signals() and call sleepq_resume_thread. Otherwise
* we have to do it here.
*/
if (!TD_IS_SLEEPING(td)) {
thread_unlock(td);
return (0);
}
wchan = td->td_wchan;
MPASS(wchan != NULL);
sq = sleepq_lookup(wchan);
MPASS(sq != NULL);
/* Thread is asleep on sleep queue sq, so wake it up. */
return (sleepq_resume_thread(sq, td, 0, 0));
}
void
sleepq_chains_remove_matching(bool (*matches)(struct thread *))
{
struct sleepqueue_chain *sc;
struct sleepqueue *sq, *sq1;
int i, wakeup_swapper;
wakeup_swapper = 0;
for (sc = &sleepq_chains[0]; sc < sleepq_chains + SC_TABLESIZE; ++sc) {
if (LIST_EMPTY(&sc->sc_queues)) {
continue;
}
mtx_lock_spin(&sc->sc_lock);
LIST_FOREACH_SAFE(sq, &sc->sc_queues, sq_hash, sq1) {
for (i = 0; i < NR_SLEEPQS; ++i) {
wakeup_swapper |= sleepq_remove_matching(sq, i,
matches, 0);
}
}
mtx_unlock_spin(&sc->sc_lock);
}
if (wakeup_swapper) {
kick_proc0();
}
}
/*
* Prints the stacks of all threads presently sleeping on wchan/queue to
* the sbuf sb. Sets count_stacks_printed to the number of stacks actually
* printed. Typically, this will equal the number of threads sleeping on the
* queue, but may be less if sb overflowed before all stacks were printed.
*/
#ifdef STACK
int
-sleepq_sbuf_print_stacks(struct sbuf *sb, void *wchan, int queue,
+sleepq_sbuf_print_stacks(struct sbuf *sb, const void *wchan, int queue,
int *count_stacks_printed)
{
struct thread *td, *td_next;
struct sleepqueue *sq;
struct stack **st;
struct sbuf **td_infos;
int i, stack_idx, error, stacks_to_allocate;
bool finished;
error = 0;
finished = false;
KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
MPASS((queue >= 0) && (queue < NR_SLEEPQS));
stacks_to_allocate = 10;
for (i = 0; i < 3 && !finished ; i++) {
/* We cannot malloc while holding the queue's spinlock, so
* we do our mallocs now, and hope it is enough. If it
* isn't, we will free these, drop the lock, malloc more,
* and try again, up to a point. After that point we will
* give up and report ENOMEM. We also cannot write to sb
* during this time since the client may have set the
* SBUF_AUTOEXTEND flag on their sbuf, which could cause a
* malloc as we print to it. So we defer actually printing
* to sb until after we drop the spinlock.
*/
/* Where we will store the stacks. */
st = malloc(sizeof(struct stack *) * stacks_to_allocate,
M_TEMP, M_WAITOK);
for (stack_idx = 0; stack_idx < stacks_to_allocate;
stack_idx++)
st[stack_idx] = stack_create(M_WAITOK);
/* Where we will store the td name, tid, etc. */
td_infos = malloc(sizeof(struct sbuf *) * stacks_to_allocate,
M_TEMP, M_WAITOK);
for (stack_idx = 0; stack_idx < stacks_to_allocate;
stack_idx++)
td_infos[stack_idx] = sbuf_new(NULL, NULL,
MAXCOMLEN + sizeof(struct thread *) * 2 + 40,
SBUF_FIXEDLEN);
sleepq_lock(wchan);
sq = sleepq_lookup(wchan);
if (sq == NULL) {
/* This sleepq does not exist; exit and return ENOENT. */
error = ENOENT;
finished = true;
sleepq_release(wchan);
goto loop_end;
}
stack_idx = 0;
/* Save thread info */
TAILQ_FOREACH_SAFE(td, &sq->sq_blocked[queue], td_slpq,
td_next) {
if (stack_idx >= stacks_to_allocate)
goto loop_end;
/* Note the td_lock is equal to the sleepq_lock here. */
stack_save_td(st[stack_idx], td);
sbuf_printf(td_infos[stack_idx], "%d: %s %p",
td->td_tid, td->td_name, td);
++stack_idx;
}
finished = true;
sleepq_release(wchan);
/* Print the stacks */
for (i = 0; i < stack_idx; i++) {
sbuf_finish(td_infos[i]);
sbuf_printf(sb, "--- thread %s: ---\n", sbuf_data(td_infos[i]));
stack_sbuf_print(sb, st[i]);
sbuf_printf(sb, "\n");
error = sbuf_error(sb);
if (error == 0)
*count_stacks_printed = stack_idx;
}
loop_end:
if (!finished)
sleepq_release(wchan);
for (stack_idx = 0; stack_idx < stacks_to_allocate;
stack_idx++)
stack_destroy(st[stack_idx]);
for (stack_idx = 0; stack_idx < stacks_to_allocate;
stack_idx++)
sbuf_delete(td_infos[stack_idx]);
free(st, M_TEMP);
free(td_infos, M_TEMP);
stacks_to_allocate *= 10;
}
if (!finished && error == 0)
error = ENOMEM;
return (error);
}
#endif
#ifdef SLEEPQUEUE_PROFILING
#define SLEEPQ_PROF_LOCATIONS 1024
#define SLEEPQ_SBUFSIZE 512
struct sleepq_prof {
LIST_ENTRY(sleepq_prof) sp_link;
const char *sp_wmesg;
long sp_count;
};
LIST_HEAD(sqphead, sleepq_prof);
struct sqphead sleepq_prof_free;
struct sqphead sleepq_hash[SC_TABLESIZE];
static struct sleepq_prof sleepq_profent[SLEEPQ_PROF_LOCATIONS];
static struct mtx sleepq_prof_lock;
MTX_SYSINIT(sleepq_prof_lock, &sleepq_prof_lock, "sleepq_prof", MTX_SPIN);
static void
sleepq_profile(const char *wmesg)
{
struct sleepq_prof *sp;
mtx_lock_spin(&sleepq_prof_lock);
if (prof_enabled == 0)
goto unlock;
LIST_FOREACH(sp, &sleepq_hash[SC_HASH(wmesg)], sp_link)
if (sp->sp_wmesg == wmesg)
goto done;
sp = LIST_FIRST(&sleepq_prof_free);
if (sp == NULL)
goto unlock;
sp->sp_wmesg = wmesg;
LIST_REMOVE(sp, sp_link);
LIST_INSERT_HEAD(&sleepq_hash[SC_HASH(wmesg)], sp, sp_link);
done:
sp->sp_count++;
unlock:
mtx_unlock_spin(&sleepq_prof_lock);
return;
}
static void
sleepq_prof_reset(void)
{
struct sleepq_prof *sp;
int enabled;
int i;
mtx_lock_spin(&sleepq_prof_lock);
enabled = prof_enabled;
prof_enabled = 0;
for (i = 0; i < SC_TABLESIZE; i++)
LIST_INIT(&sleepq_hash[i]);
LIST_INIT(&sleepq_prof_free);
for (i = 0; i < SLEEPQ_PROF_LOCATIONS; i++) {
sp = &sleepq_profent[i];
sp->sp_wmesg = NULL;
sp->sp_count = 0;
LIST_INSERT_HEAD(&sleepq_prof_free, sp, sp_link);
}
prof_enabled = enabled;
mtx_unlock_spin(&sleepq_prof_lock);
}
static int
enable_sleepq_prof(SYSCTL_HANDLER_ARGS)
{
int error, v;
v = prof_enabled;
error = sysctl_handle_int(oidp, &v, v, req);
if (error)
return (error);
if (req->newptr == NULL)
return (error);
if (v == prof_enabled)
return (0);
if (v == 1)
sleepq_prof_reset();
mtx_lock_spin(&sleepq_prof_lock);
prof_enabled = !!v;
mtx_unlock_spin(&sleepq_prof_lock);
return (0);
}
static int
reset_sleepq_prof_stats(SYSCTL_HANDLER_ARGS)
{
int error, v;
v = 0;
error = sysctl_handle_int(oidp, &v, 0, req);
if (error)
return (error);
if (req->newptr == NULL)
return (error);
if (v == 0)
return (0);
sleepq_prof_reset();
return (0);
}
static int
dump_sleepq_prof_stats(SYSCTL_HANDLER_ARGS)
{
struct sleepq_prof *sp;
struct sbuf *sb;
int enabled;
int error;
int i;
error = sysctl_wire_old_buffer(req, 0);
if (error != 0)
return (error);
sb = sbuf_new_for_sysctl(NULL, NULL, SLEEPQ_SBUFSIZE, req);
sbuf_printf(sb, "\nwmesg\tcount\n");
enabled = prof_enabled;
mtx_lock_spin(&sleepq_prof_lock);
prof_enabled = 0;
mtx_unlock_spin(&sleepq_prof_lock);
for (i = 0; i < SC_TABLESIZE; i++) {
LIST_FOREACH(sp, &sleepq_hash[i], sp_link) {
sbuf_printf(sb, "%s\t%ld\n",
sp->sp_wmesg, sp->sp_count);
}
}
mtx_lock_spin(&sleepq_prof_lock);
prof_enabled = enabled;
mtx_unlock_spin(&sleepq_prof_lock);
error = sbuf_finish(sb);
sbuf_delete(sb);
return (error);
}
SYSCTL_PROC(_debug_sleepq, OID_AUTO, stats, CTLTYPE_STRING | CTLFLAG_RD,
NULL, 0, dump_sleepq_prof_stats, "A", "Sleepqueue profiling statistics");
SYSCTL_PROC(_debug_sleepq, OID_AUTO, reset, CTLTYPE_INT | CTLFLAG_RW,
NULL, 0, reset_sleepq_prof_stats, "I",
"Reset sleepqueue profiling statistics");
SYSCTL_PROC(_debug_sleepq, OID_AUTO, enable, CTLTYPE_INT | CTLFLAG_RW,
NULL, 0, enable_sleepq_prof, "I", "Enable sleepqueue profiling");
#endif
#ifdef DDB
DB_SHOW_COMMAND(sleepq, db_show_sleepqueue)
{
struct sleepqueue_chain *sc;
struct sleepqueue *sq;
#ifdef INVARIANTS
struct lock_object *lock;
#endif
struct thread *td;
void *wchan;
int i;
if (!have_addr)
return;
/*
* First, see if there is an active sleep queue for the wait channel
* indicated by the address.
*/
wchan = (void *)addr;
sc = SC_LOOKUP(wchan);
LIST_FOREACH(sq, &sc->sc_queues, sq_hash)
if (sq->sq_wchan == wchan)
goto found;
/*
* Second, see if there is an active sleep queue at the address
* indicated.
*/
for (i = 0; i < SC_TABLESIZE; i++)
LIST_FOREACH(sq, &sleepq_chains[i].sc_queues, sq_hash) {
if (sq == (struct sleepqueue *)addr)
goto found;
}
db_printf("Unable to locate a sleep queue via %p\n", (void *)addr);
return;
found:
db_printf("Wait channel: %p\n", sq->sq_wchan);
db_printf("Queue type: %d\n", sq->sq_type);
#ifdef INVARIANTS
if (sq->sq_lock) {
lock = sq->sq_lock;
db_printf("Associated Interlock: %p - (%s) %s\n", lock,
LOCK_CLASS(lock)->lc_name, lock->lo_name);
}
#endif
db_printf("Blocked threads:\n");
for (i = 0; i < NR_SLEEPQS; i++) {
db_printf("\nQueue[%d]:\n", i);
if (TAILQ_EMPTY(&sq->sq_blocked[i]))
db_printf("\tempty\n");
else
TAILQ_FOREACH(td, &sq->sq_blocked[i],
td_slpq) {
db_printf("\t%p (tid %d, pid %d, \"%s\")\n", td,
td->td_tid, td->td_proc->p_pid,
td->td_name);
}
db_printf("(expected: %u)\n", sq->sq_blockedcnt[i]);
}
}
/* Alias 'show sleepqueue' to 'show sleepq'. */
DB_SHOW_ALIAS(sleepqueue, db_show_sleepqueue);
#endif
Index: head/sys/sys/proc.h
===================================================================
--- head/sys/sys/proc.h (revision 356056)
+++ head/sys/sys/proc.h (revision 356057)
@@ -1,1234 +1,1234 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1986, 1989, 1991, 1993
* The Regents of the University of California. All rights reserved.
* (c) UNIX System Laboratories, Inc.
* All or some portions of this file are derived from material licensed
* to the University of California by American Telephone and Telegraph
* Co. or Unix System Laboratories, Inc. and are reproduced herein with
* the permission of UNIX System Laboratories, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)proc.h 8.15 (Berkeley) 5/19/95
* $FreeBSD$
*/
#ifndef _SYS_PROC_H_
#define _SYS_PROC_H_
#include <sys/callout.h> /* For struct callout. */
#include <sys/event.h> /* For struct klist. */
#ifdef _KERNEL
#include <sys/_eventhandler.h>
#endif
#include <sys/condvar.h>
#ifndef _KERNEL
#include <sys/filedesc.h>
#endif
#include <sys/queue.h>
#include <sys/_lock.h>
#include <sys/lock_profile.h>
#include <sys/_mutex.h>
#include <sys/osd.h>
#include <sys/priority.h>
#include <sys/rtprio.h> /* XXX. */
#include <sys/runq.h>
#include <sys/resource.h>
#include <sys/sigio.h>
#include <sys/signal.h>
#include <sys/signalvar.h>
#ifndef _KERNEL
#include <sys/time.h> /* For structs itimerval, timeval. */
#else
#include <sys/pcpu.h>
#include <sys/systm.h>
#endif
#include <sys/ucontext.h>
#include <sys/ucred.h>
#include <sys/types.h>
#include <sys/_domainset.h>
#include <machine/proc.h> /* Machine-dependent proc substruct. */
#ifdef _KERNEL
#include <machine/cpu.h>
#endif
/*
* One structure allocated per session.
*
* List of locks
* (m) locked by s_mtx mtx
* (e) locked by proctree_lock sx
* (c) const until freeing
*/
struct session {
u_int s_count; /* Ref cnt; pgrps in session - atomic. */
struct proc *s_leader; /* (m + e) Session leader. */
struct vnode *s_ttyvp; /* (m) Vnode of controlling tty. */
struct cdev_priv *s_ttydp; /* (m) Device of controlling tty. */
struct tty *s_ttyp; /* (e) Controlling tty. */
pid_t s_sid; /* (c) Session ID. */
/* (m) Setlogin() name: */
char s_login[roundup(MAXLOGNAME, sizeof(long))];
struct mtx s_mtx; /* Mutex to protect members. */
};
/*
* One structure allocated per process group.
*
* List of locks
* (m) locked by pg_mtx mtx
* (e) locked by proctree_lock sx
* (c) const until freeing
*/
struct pgrp {
LIST_ENTRY(pgrp) pg_hash; /* (e) Hash chain. */
LIST_HEAD(, proc) pg_members; /* (m + e) Pointer to pgrp members. */
struct session *pg_session; /* (c) Pointer to session. */
struct sigiolst pg_sigiolst; /* (m) List of sigio sources. */
pid_t pg_id; /* (c) Process group id. */
int pg_jobc; /* (m) Job control process count. */
struct mtx pg_mtx; /* Mutex to protect members */
};
/*
* pargs, used to hold a copy of the command line, if it had a sane length.
*/
struct pargs {
u_int ar_ref; /* Reference count. */
u_int ar_length; /* Length. */
u_char ar_args[1]; /* Arguments. */
};
/*-
* Description of a process.
*
* This structure contains the information needed to manage a thread of
* control, known in UN*X as a process; it has references to substructures
* containing descriptions of things that the process uses, but may share
* with related processes. The process structure and the substructures
* are always addressable except for those marked "(CPU)" below,
* which might be addressable only on a processor on which the process
* is running.
*
* Below is a key of locks used to protect each member of struct proc. The
* lock is indicated by a reference to a specific character in parens in the
* associated comment.
* * - not yet protected
* a - only touched by curproc or parent during fork/wait
* b - created at fork, never changes
* (exception aiods switch vmspaces, but they are also
* marked 'P_SYSTEM' so hopefully it will be left alone)
* c - locked by proc mtx
* d - locked by allproc_lock lock
* e - locked by proctree_lock lock
* f - session mtx
* g - process group mtx
* h - callout_lock mtx
* i - by curproc or the master session mtx
* j - locked by proc slock
* k - only accessed by curthread
* k*- only accessed by curthread and from an interrupt
* kx- only accessed by curthread and by debugger
* l - the attaching proc or attaching proc parent
* m - Giant
* n - not locked, lazy
* o - ktrace lock
* q - td_contested lock
* r - p_peers lock
* s - see sleepq_switch(), sleeping_on_old_rtc(), and sleep(9)
* t - thread lock
* u - process stat lock
* w - process timer lock
* x - created at fork, only changes during single threading in exec
* y - created at first aio, doesn't change until exit or exec at which
* point we are single-threaded and only curthread changes it
* z - zombie threads lock
*
* If the locking key specifies two identifiers (for example, p_pptr) then
* either lock is sufficient for read access, but both locks must be held
* for write access.
*/
struct cpuset;
struct filecaps;
struct filemon;
struct kaioinfo;
struct kaudit_record;
struct kcov_info;
struct kdtrace_proc;
struct kdtrace_thread;
struct mqueue_notifier;
struct nlminfo;
struct p_sched;
struct proc;
struct procdesc;
struct racct;
struct sbuf;
struct sleepqueue;
struct socket;
struct syscall_args;
struct td_sched;
struct thread;
struct trapframe;
struct turnstile;
struct vm_map;
struct vm_map_entry;
struct epoch_tracker;
/*
* XXX: Does this belong in resource.h or resourcevar.h instead?
* Resource usage extension. The times in rusage structs in the kernel are
* never up to date. The actual times are kept as runtimes and tick counts
* (with control info in the "previous" times), and are converted when
* userland asks for rusage info. Backwards compatibility prevents putting
* this directly in the user-visible rusage struct.
*
* Locking for p_rux: (cu) means (u) for p_rux and (c) for p_crux.
* Locking for td_rux: (t) for all fields.
*/
struct rusage_ext {
uint64_t rux_runtime; /* (cu) Real time. */
uint64_t rux_uticks; /* (cu) Statclock hits in user mode. */
uint64_t rux_sticks; /* (cu) Statclock hits in sys mode. */
uint64_t rux_iticks; /* (cu) Statclock hits in intr mode. */
uint64_t rux_uu; /* (c) Previous user time in usec. */
uint64_t rux_su; /* (c) Previous sys time in usec. */
uint64_t rux_tu; /* (c) Previous total time in usec. */
};
/*
* Kernel runnable context (thread).
* This is what is put to sleep and reactivated.
* Thread context. Processes may have multiple threads.
*/
struct thread {
struct mtx *volatile td_lock; /* replaces sched lock */
struct proc *td_proc; /* (*) Associated process. */
TAILQ_ENTRY(thread) td_plist; /* (*) All threads in this proc. */
TAILQ_ENTRY(thread) td_runq; /* (t) Run queue. */
TAILQ_ENTRY(thread) td_slpq; /* (t) Sleep queue. */
TAILQ_ENTRY(thread) td_lockq; /* (t) Lock queue. */
LIST_ENTRY(thread) td_hash; /* (d) Hash chain. */
struct cpuset *td_cpuset; /* (t) CPU affinity mask. */
struct domainset_ref td_domain; /* (a) NUMA policy */
struct seltd *td_sel; /* Select queue/channel. */
struct sleepqueue *td_sleepqueue; /* (k) Associated sleep queue. */
struct turnstile *td_turnstile; /* (k) Associated turnstile. */
struct rl_q_entry *td_rlqe; /* (k) Associated range lock entry. */
struct umtx_q *td_umtxq; /* (c?) Link for when we're blocked. */
lwpid_t td_tid; /* (b) Thread ID. */
sigqueue_t td_sigqueue; /* (c) Sigs arrived, not delivered. */
#define td_siglist td_sigqueue.sq_signals
u_char td_lend_user_pri; /* (t) Lend user pri. */
/* Cleared during fork1() */
#define td_startzero td_flags
int td_flags; /* (t) TDF_* flags. */
int td_inhibitors; /* (t) Why can not run. */
int td_pflags; /* (k) Private thread (TDP_*) flags. */
int td_dupfd; /* (k) Ret value from fdopen. XXX */
int td_sqqueue; /* (t) Sleepqueue queue blocked on. */
- void *td_wchan; /* (t) Sleep address. */
+ const void *td_wchan; /* (t) Sleep address. */
const char *td_wmesg; /* (t) Reason for sleep. */
volatile u_char td_owepreempt; /* (k*) Preempt on last critical_exit */
u_char td_tsqueue; /* (t) Turnstile queue blocked on. */
short td_locks; /* (k) Debug: count of non-spin locks */
short td_rw_rlocks; /* (k) Count of rwlock read locks. */
short td_sx_slocks; /* (k) Count of sx shared locks. */
short td_lk_slocks; /* (k) Count of lockmgr shared locks. */
short td_stopsched; /* (k) Scheduler stopped. */
struct turnstile *td_blocked; /* (t) Lock thread is blocked on. */
const char *td_lockname; /* (t) Name of lock blocked on. */
LIST_HEAD(, turnstile) td_contested; /* (q) Contested locks. */
struct lock_list_entry *td_sleeplocks; /* (k) Held sleep locks. */
int td_intr_nesting_level; /* (k) Interrupt recursion. */
int td_pinned; /* (k) Temporary cpu pin count. */
struct ucred *td_ucred; /* (k) Reference to credentials. */
struct plimit *td_limit; /* (k) Resource limits. */
int td_slptick; /* (t) Time at sleep. */
int td_blktick; /* (t) Time spent blocked. */
int td_swvoltick; /* (t) Time at last SW_VOL switch. */
int td_swinvoltick; /* (t) Time at last SW_INVOL switch. */
u_int td_cow; /* (*) Number of copy-on-write faults */
struct rusage td_ru; /* (t) rusage information. */
struct rusage_ext td_rux; /* (t) Internal rusage information. */
uint64_t td_incruntime; /* (t) Cpu ticks to transfer to proc. */
uint64_t td_runtime; /* (t) How many cpu ticks we've run. */
u_int td_pticks; /* (t) Statclock hits for profiling */
u_int td_sticks; /* (t) Statclock hits in system mode. */
u_int td_iticks; /* (t) Statclock hits in intr mode. */
u_int td_uticks; /* (t) Statclock hits in user mode. */
int td_intrval; /* (t) Return value for sleepq. */
sigset_t td_oldsigmask; /* (k) Saved mask from pre sigpause. */
volatile u_int td_generation; /* (k) For detection of preemption */
stack_t td_sigstk; /* (k) Stack ptr and on-stack flag. */
int td_xsig; /* (c) Signal for ptrace */
u_long td_profil_addr; /* (k) Temporary addr until AST. */
u_int td_profil_ticks; /* (k) Temporary ticks until AST. */
char td_name[MAXCOMLEN + 1]; /* (*) Thread name. */
struct file *td_fpop; /* (k) file referencing cdev under op */
int td_dbgflags; /* (c) Userland debugger flags */
siginfo_t td_si; /* (c) For debugger or core file */
int td_ng_outbound; /* (k) Thread entered ng from above. */
struct osd td_osd; /* (k) Object specific data. */
struct vm_map_entry *td_map_def_user; /* (k) Deferred entries. */
pid_t td_dbg_forked; /* (c) Child pid for debugger. */
u_int td_vp_reserv; /* (k) Count of reserved vnodes. */
u_int td_no_sleeping; /* (k) Sleeping disabled count. */
void *td_su; /* (k) FFS SU private */
sbintime_t td_sleeptimo; /* (t) Sleep timeout. */
int td_rtcgen; /* (s) rtc_generation of abs. sleep */
int td_errno; /* (k) Error from last syscall. */
size_t td_vslock_sz; /* (k) amount of vslock-ed space */
struct kcov_info *td_kcov_info; /* (*) Kernel code coverage data */
#define td_endzero td_sigmask
/* Copied during fork1() or create_thread(). */
#define td_startcopy td_endzero
sigset_t td_sigmask; /* (c) Current signal mask. */
u_char td_rqindex; /* (t) Run queue index. */
u_char td_base_pri; /* (t) Thread base kernel priority. */
u_char td_priority; /* (t) Thread active priority. */
u_char td_pri_class; /* (t) Scheduling class. */
u_char td_user_pri; /* (t) User pri from estcpu and nice. */
u_char td_base_user_pri; /* (t) Base user pri */
u_char td_pre_epoch_prio; /* (k) User pri on entry to epoch */
uintptr_t td_rb_list; /* (k) Robust list head. */
uintptr_t td_rbp_list; /* (k) Robust priv list head. */
uintptr_t td_rb_inact; /* (k) Current in-action mutex loc. */
struct syscall_args td_sa; /* (kx) Syscall parameters. Copied on
fork for child tracing. */
#define td_endcopy td_pcb
/*
* Fields that must be manually set in fork1() or create_thread()
* or already have been set in the allocator, constructor, etc.
*/
struct pcb *td_pcb; /* (k) Kernel VA of pcb and kstack. */
enum td_states {
TDS_INACTIVE = 0x0,
TDS_INHIBITED,
TDS_CAN_RUN,
TDS_RUNQ,
TDS_RUNNING
} td_state; /* (t) thread state */
union {
register_t tdu_retval[2];
off_t tdu_off;
} td_uretoff; /* (k) Syscall aux returns. */
#define td_retval td_uretoff.tdu_retval
u_int td_cowgen; /* (k) Generation of COW pointers. */
/* LP64 hole */
struct callout td_slpcallout; /* (h) Callout for sleep. */
struct trapframe *td_frame; /* (k) */
struct vm_object *td_kstack_obj;/* (a) Kstack object. */
vm_offset_t td_kstack; /* (a) Kernel VA of kstack. */
int td_kstack_pages; /* (a) Size of the kstack. */
volatile u_int td_critnest; /* (k*) Critical section nest level. */
struct mdthread td_md; /* (k) Any machine-dependent fields. */
struct kaudit_record *td_ar; /* (k) Active audit record, if any. */
struct lpohead td_lprof[2]; /* (a) lock profiling objects. */
struct kdtrace_thread *td_dtrace; /* (*) DTrace-specific data. */
struct vnet *td_vnet; /* (k) Effective vnet. */
const char *td_vnet_lpush; /* (k) Debugging vnet push / pop. */
struct trapframe *td_intr_frame;/* (k) Frame of the current irq */
struct proc *td_rfppwait_p; /* (k) The vforked child */
struct vm_page **td_ma; /* (k) uio pages held */
int td_ma_cnt; /* (k) size of *td_ma */
/* LP64 hole */
void *td_emuldata; /* Emulator state data */
int td_lastcpu; /* (t) Last cpu we were on. */
int td_oncpu; /* (t) Which cpu we are on. */
void *td_lkpi_task; /* LinuxKPI task struct pointer */
int td_pmcpend;
#ifdef EPOCH_TRACE
SLIST_HEAD(, epoch_tracker) td_epochs;
#endif
};
struct thread0_storage {
struct thread t0st_thread;
uint64_t t0st_sched[10];
};
struct mtx *thread_lock_block(struct thread *);
void thread_lock_block_wait(struct thread *);
void thread_lock_set(struct thread *, struct mtx *);
void thread_lock_unblock(struct thread *, struct mtx *);
#define THREAD_LOCK_ASSERT(td, type) \
mtx_assert((td)->td_lock, (type))
#define THREAD_LOCK_BLOCKED_ASSERT(td, type) \
do { \
struct mtx *__m = (td)->td_lock; \
if (__m != &blocked_lock) \
mtx_assert(__m, (type)); \
} while (0)
#ifdef INVARIANTS
#define THREAD_LOCKPTR_ASSERT(td, lock) \
do { \
struct mtx *__m; \
__m = (td)->td_lock; \
KASSERT(__m == (lock), \
("Thread %p lock %p does not match %p", td, __m, (lock))); \
} while (0)
#define THREAD_LOCKPTR_BLOCKED_ASSERT(td, lock) \
do { \
struct mtx *__m; \
__m = (td)->td_lock; \
KASSERT(__m == (lock) || __m == &blocked_lock, \
("Thread %p lock %p does not match %p", td, __m, (lock))); \
} while (0)
#define TD_LOCKS_INC(td) ((td)->td_locks++)
#define TD_LOCKS_DEC(td) do { \
KASSERT(SCHEDULER_STOPPED_TD(td) || (td)->td_locks > 0, \
("thread %p owns no locks", (td))); \
(td)->td_locks--; \
} while (0)
#else
#define THREAD_LOCKPTR_ASSERT(td, lock)
#define THREAD_LOCKPTR_BLOCKED_ASSERT(td, lock)
#define TD_LOCKS_INC(td)
#define TD_LOCKS_DEC(td)
#endif
/*
* Flags kept in td_flags:
* To change these you MUST have the scheduler lock.
*/
#define TDF_BORROWING 0x00000001 /* Thread is borrowing pri from another. */
#define TDF_INPANIC 0x00000002 /* Caused a panic, let it drive crashdump. */
#define TDF_INMEM 0x00000004 /* Thread's stack is in memory. */
#define TDF_SINTR 0x00000008 /* Sleep is interruptible. */
#define TDF_TIMEOUT 0x00000010 /* Timing out during sleep. */
#define TDF_IDLETD 0x00000020 /* This is a per-CPU idle thread. */
#define TDF_CANSWAP 0x00000040 /* Thread can be swapped. */
#define TDF_UNUSED80 0x00000080 /* unused. */
#define TDF_KTH_SUSP 0x00000100 /* kthread is suspended */
#define TDF_ALLPROCSUSP 0x00000200 /* suspended by SINGLE_ALLPROC */
#define TDF_BOUNDARY 0x00000400 /* Thread suspended at user boundary */
#define TDF_ASTPENDING 0x00000800 /* Thread has some asynchronous events. */
#define TDF_UNUSED12 0x00001000 /* --available-- */
#define TDF_SBDRY 0x00002000 /* Stop only on usermode boundary. */
#define TDF_UPIBLOCKED 0x00004000 /* Thread blocked on user PI mutex. */
#define TDF_NEEDSUSPCHK 0x00008000 /* Thread may need to suspend. */
#define TDF_NEEDRESCHED 0x00010000 /* Thread needs to yield. */
#define TDF_NEEDSIGCHK 0x00020000 /* Thread may need signal delivery. */
#define TDF_NOLOAD 0x00040000 /* Ignore during load avg calculations. */
#define TDF_SERESTART 0x00080000 /* ERESTART on stop attempts. */
#define TDF_THRWAKEUP 0x00100000 /* Libthr thread must not suspend itself. */
#define TDF_SEINTR 0x00200000 /* EINTR on stop attempts. */
#define TDF_SWAPINREQ 0x00400000 /* Swapin request due to wakeup. */
#define TDF_UNUSED23 0x00800000 /* --available-- */
#define TDF_SCHED0 0x01000000 /* Reserved for scheduler private use */
#define TDF_SCHED1 0x02000000 /* Reserved for scheduler private use */
#define TDF_SCHED2 0x04000000 /* Reserved for scheduler private use */
#define TDF_SCHED3 0x08000000 /* Reserved for scheduler private use */
#define TDF_ALRMPEND 0x10000000 /* Pending SIGVTALRM needs to be posted. */
#define TDF_PROFPEND 0x20000000 /* Pending SIGPROF needs to be posted. */
#define TDF_MACPEND 0x40000000 /* AST-based MAC event pending. */
/* Userland debug flags */
#define TDB_SUSPEND 0x00000001 /* Thread is suspended by debugger */
#define TDB_XSIG 0x00000002 /* Thread is exchanging signal under trace */
#define TDB_USERWR 0x00000004 /* Debugger modified memory or registers */
#define TDB_SCE 0x00000008 /* Thread performs syscall enter */
#define TDB_SCX 0x00000010 /* Thread performs syscall exit */
#define TDB_EXEC 0x00000020 /* TDB_SCX from exec(2) family */
#define TDB_FORK 0x00000040 /* TDB_SCX from fork(2) that created new
process */
#define TDB_STOPATFORK 0x00000080 /* Stop at the return from fork (child
only) */
#define TDB_CHILD 0x00000100 /* New child indicator for ptrace() */
#define TDB_BORN 0x00000200 /* New LWP indicator for ptrace() */
#define TDB_EXIT 0x00000400 /* Exiting LWP indicator for ptrace() */
#define TDB_VFORK 0x00000800 /* vfork indicator for ptrace() */
#define TDB_FSTP 0x00001000 /* The thread is PT_ATTACH leader */
#define TDB_STEP 0x00002000 /* (x86) PSL_T set for PT_STEP */
/*
* "Private" flags kept in td_pflags:
* These are only written by curthread and thus need no locking.
*/
#define TDP_OLDMASK 0x00000001 /* Need to restore mask after suspend. */
#define TDP_INKTR 0x00000002 /* Thread is currently in KTR code. */
#define TDP_INKTRACE 0x00000004 /* Thread is currently in KTRACE code. */
#define TDP_BUFNEED 0x00000008 /* Do not recurse into the buf flush */
#define TDP_COWINPROGRESS 0x00000010 /* Snapshot copy-on-write in progress. */
#define TDP_ALTSTACK 0x00000020 /* Have alternate signal stack. */
#define TDP_DEADLKTREAT 0x00000040 /* Lock acquisition - deadlock treatment. */
#define TDP_NOFAULTING 0x00000080 /* Do not handle page faults. */
#define TDP_UNUSED9 0x00000100 /* --available-- */
#define TDP_OWEUPC 0x00000200 /* Call addupc() at next AST. */
#define TDP_ITHREAD 0x00000400 /* Thread is an interrupt thread. */
#define TDP_SYNCIO 0x00000800 /* Local override, disable async i/o. */
#define TDP_SCHED1 0x00001000 /* Reserved for scheduler private use */
#define TDP_SCHED2 0x00002000 /* Reserved for scheduler private use */
#define TDP_SCHED3 0x00004000 /* Reserved for scheduler private use */
#define TDP_SCHED4 0x00008000 /* Reserved for scheduler private use */
#define TDP_GEOM 0x00010000 /* Settle GEOM before finishing syscall */
#define TDP_SOFTDEP 0x00020000 /* Stuck processing softdep worklist */
#define TDP_NORUNNINGBUF 0x00040000 /* Ignore runningbufspace check */
#define TDP_WAKEUP 0x00080000 /* Don't sleep in umtx cond_wait */
#define TDP_INBDFLUSH 0x00100000 /* Already in BO_BDFLUSH, do not recurse */
#define TDP_KTHREAD 0x00200000 /* This is an official kernel thread */
#define TDP_CALLCHAIN 0x00400000 /* Capture thread's callchain */
#define TDP_IGNSUSP 0x00800000 /* Permission to ignore the MNTK_SUSPEND* */
#define TDP_AUDITREC 0x01000000 /* Audit record pending on thread */
#define TDP_RFPPWAIT 0x02000000 /* Handle RFPPWAIT on syscall exit */
#define TDP_RESETSPUR 0x04000000 /* Reset spurious page fault history. */
#define TDP_NERRNO 0x08000000 /* Last errno is already in td_errno */
#define TDP_UIOHELD 0x10000000 /* Current uio has pages held in td_ma */
#define TDP_FORKING 0x20000000 /* Thread is being created through fork() */
#define TDP_EXECVMSPC 0x40000000 /* Execve destroyed old vmspace */
/*
* Reasons that the current thread can not be run yet.
* More than one may apply.
*/
#define TDI_SUSPENDED 0x0001 /* On suspension queue. */
#define TDI_SLEEPING 0x0002 /* Actually asleep! (tricky). */
#define TDI_SWAPPED 0x0004 /* Stack not in mem. Bad juju if run. */
#define TDI_LOCK 0x0008 /* Stopped on a lock. */
#define TDI_IWAIT 0x0010 /* Awaiting interrupt. */
#define TD_IS_SLEEPING(td) ((td)->td_inhibitors & TDI_SLEEPING)
#define TD_ON_SLEEPQ(td) ((td)->td_wchan != NULL)
#define TD_IS_SUSPENDED(td) ((td)->td_inhibitors & TDI_SUSPENDED)
#define TD_IS_SWAPPED(td) ((td)->td_inhibitors & TDI_SWAPPED)
#define TD_ON_LOCK(td) ((td)->td_inhibitors & TDI_LOCK)
#define TD_AWAITING_INTR(td) ((td)->td_inhibitors & TDI_IWAIT)
#define TD_IS_RUNNING(td) ((td)->td_state == TDS_RUNNING)
#define TD_ON_RUNQ(td) ((td)->td_state == TDS_RUNQ)
#define TD_CAN_RUN(td) ((td)->td_state == TDS_CAN_RUN)
#define TD_IS_INHIBITED(td) ((td)->td_state == TDS_INHIBITED)
#define TD_ON_UPILOCK(td) ((td)->td_flags & TDF_UPIBLOCKED)
#define TD_IS_IDLETHREAD(td) ((td)->td_flags & TDF_IDLETD)
#define TD_CAN_ABORT(td) (TD_ON_SLEEPQ((td)) && \
((td)->td_flags & TDF_SINTR) != 0)
#define KTDSTATE(td) \
(((td)->td_inhibitors & TDI_SLEEPING) != 0 ? "sleep" : \
((td)->td_inhibitors & TDI_SUSPENDED) != 0 ? "suspended" : \
((td)->td_inhibitors & TDI_SWAPPED) != 0 ? "swapped" : \
((td)->td_inhibitors & TDI_LOCK) != 0 ? "blocked" : \
((td)->td_inhibitors & TDI_IWAIT) != 0 ? "iwait" : "yielding")
#define TD_SET_INHIB(td, inhib) do { \
(td)->td_state = TDS_INHIBITED; \
(td)->td_inhibitors |= (inhib); \
} while (0)
#define TD_CLR_INHIB(td, inhib) do { \
if (((td)->td_inhibitors & (inhib)) && \
(((td)->td_inhibitors &= ~(inhib)) == 0)) \
(td)->td_state = TDS_CAN_RUN; \
} while (0)
#define TD_SET_SLEEPING(td) TD_SET_INHIB((td), TDI_SLEEPING)
#define TD_SET_SWAPPED(td) TD_SET_INHIB((td), TDI_SWAPPED)
#define TD_SET_LOCK(td) TD_SET_INHIB((td), TDI_LOCK)
#define TD_SET_SUSPENDED(td) TD_SET_INHIB((td), TDI_SUSPENDED)
#define TD_SET_IWAIT(td) TD_SET_INHIB((td), TDI_IWAIT)
#define TD_SET_EXITING(td) TD_SET_INHIB((td), TDI_EXITING)
#define TD_CLR_SLEEPING(td) TD_CLR_INHIB((td), TDI_SLEEPING)
#define TD_CLR_SWAPPED(td) TD_CLR_INHIB((td), TDI_SWAPPED)
#define TD_CLR_LOCK(td) TD_CLR_INHIB((td), TDI_LOCK)
#define TD_CLR_SUSPENDED(td) TD_CLR_INHIB((td), TDI_SUSPENDED)
#define TD_CLR_IWAIT(td) TD_CLR_INHIB((td), TDI_IWAIT)
#define TD_SET_RUNNING(td) (td)->td_state = TDS_RUNNING
#define TD_SET_RUNQ(td) (td)->td_state = TDS_RUNQ
#define TD_SET_CAN_RUN(td) (td)->td_state = TDS_CAN_RUN
#define TD_SBDRY_INTR(td) \
(((td)->td_flags & (TDF_SEINTR | TDF_SERESTART)) != 0)
#define TD_SBDRY_ERRNO(td) \
(((td)->td_flags & TDF_SEINTR) != 0 ? EINTR : ERESTART)
/*
* Process structure.
*/
struct proc {
LIST_ENTRY(proc) p_list; /* (d) List of all processes. */
TAILQ_HEAD(, thread) p_threads; /* (c) all threads. */
struct mtx p_slock; /* process spin lock */
struct ucred *p_ucred; /* (c) Process owner's identity. */
struct filedesc *p_fd; /* (b) Open files. */
struct filedesc_to_leader *p_fdtol; /* (b) Tracking node */
struct pstats *p_stats; /* (b) Accounting/statistics (CPU). */
struct plimit *p_limit; /* (c) Resource limits. */
struct callout p_limco; /* (c) Limit callout handle */
struct sigacts *p_sigacts; /* (x) Signal actions, state (CPU). */
int p_flag; /* (c) P_* flags. */
int p_flag2; /* (c) P2_* flags. */
enum p_states {
PRS_NEW = 0, /* In creation */
PRS_NORMAL, /* threads can be run. */
PRS_ZOMBIE
} p_state; /* (j/c) Process status. */
pid_t p_pid; /* (b) Process identifier. */
LIST_ENTRY(proc) p_hash; /* (d) Hash chain. */
LIST_ENTRY(proc) p_pglist; /* (g + e) List of processes in pgrp. */
struct proc *p_pptr; /* (c + e) Pointer to parent process. */
LIST_ENTRY(proc) p_sibling; /* (e) List of sibling processes. */
LIST_HEAD(, proc) p_children; /* (e) Pointer to list of children. */
struct proc *p_reaper; /* (e) My reaper. */
LIST_HEAD(, proc) p_reaplist; /* (e) List of my descendants
(if I am reaper). */
LIST_ENTRY(proc) p_reapsibling; /* (e) List of siblings - descendants of
the same reaper. */
struct mtx p_mtx; /* (n) Lock for this struct. */
struct mtx p_statmtx; /* Lock for the stats */
struct mtx p_itimmtx; /* Lock for the virt/prof timers */
struct mtx p_profmtx; /* Lock for the profiling */
struct ksiginfo *p_ksi; /* Locked by parent proc lock */
sigqueue_t p_sigqueue; /* (c) Sigs not delivered to a td. */
#define p_siglist p_sigqueue.sq_signals
pid_t p_oppid; /* (c + e) Real parent pid. */
/* The following fields are all zeroed upon creation in fork. */
#define p_startzero p_vmspace
struct vmspace *p_vmspace; /* (b) Address space. */
u_int p_swtick; /* (c) Tick when swapped in or out. */
u_int p_cowgen; /* (c) Generation of COW pointers. */
struct itimerval p_realtimer; /* (c) Alarm timer. */
struct rusage p_ru; /* (a) Exit information. */
struct rusage_ext p_rux; /* (cu) Internal resource usage. */
struct rusage_ext p_crux; /* (c) Internal child resource usage. */
int p_profthreads; /* (c) Num threads in addupc_task. */
volatile int p_exitthreads; /* (j) Number of threads exiting */
int p_traceflag; /* (o) Kernel trace points. */
struct vnode *p_tracevp; /* (c + o) Trace to vnode. */
struct ucred *p_tracecred; /* (o) Credentials to trace with. */
struct vnode *p_textvp; /* (b) Vnode of executable. */
u_int p_lock; /* (c) Proclock (prevent swap) count. */
struct sigiolst p_sigiolst; /* (c) List of sigio sources. */
int p_sigparent; /* (c) Signal to parent on exit. */
int p_sig; /* (n) For core dump/debugger XXX. */
u_int p_stops; /* (c) Stop event bitmask. */
u_int p_stype; /* (c) Stop event type. */
char p_step; /* (c) Process is stopped. */
u_char p_pfsflags; /* (c) Procfs flags. */
u_int p_ptevents; /* (c + e) ptrace() event mask. */
struct nlminfo *p_nlminfo; /* (?) Only used by/for lockd. */
struct kaioinfo *p_aioinfo; /* (y) ASYNC I/O info. */
struct thread *p_singlethread;/* (c + j) If single threading this is it */
int p_suspcount; /* (j) Num threads in suspended mode. */
struct thread *p_xthread; /* (c) Trap thread */
int p_boundary_count;/* (j) Num threads at user boundary */
int p_pendingcnt; /* how many signals are pending */
struct itimers *p_itimers; /* (c) POSIX interval timers. */
struct procdesc *p_procdesc; /* (e) Process descriptor, if any. */
u_int p_treeflag; /* (e) P_TREE flags */
int p_pendingexits; /* (c) Count of pending thread exits. */
struct filemon *p_filemon; /* (c) filemon-specific data. */
int p_pdeathsig; /* (c) Signal from parent on exit. */
/* End area that is zeroed on creation. */
#define p_endzero p_magic
/* The following fields are all copied upon creation in fork. */
#define p_startcopy p_endzero
u_int p_magic; /* (b) Magic number. */
int p_osrel; /* (x) osreldate for the
binary (from ELF note, if any) */
uint32_t p_fctl0; /* (x) ABI feature control, ELF note */
char p_comm[MAXCOMLEN + 1]; /* (x) Process name. */
struct sysentvec *p_sysent; /* (b) Syscall dispatch info. */
struct pargs *p_args; /* (c) Process arguments. */
rlim_t p_cpulimit; /* (c) Current CPU limit in seconds. */
signed char p_nice; /* (c) Process "nice" value. */
int p_fibnum; /* in this routing domain XXX MRT */
pid_t p_reapsubtree; /* (e) Pid of the direct child of the
reaper which spawned
our subtree. */
uint16_t p_elf_machine; /* (x) ELF machine type */
uint64_t p_elf_flags; /* (x) ELF flags */
/* End area that is copied on creation. */
#define p_endcopy p_xexit
u_int p_xexit; /* (c) Exit code. */
u_int p_xsig; /* (c) Stop/kill sig. */
struct pgrp *p_pgrp; /* (c + e) Pointer to process group. */
struct knlist *p_klist; /* (c) Knotes attached to this proc. */
int p_numthreads; /* (c) Number of threads. */
struct mdproc p_md; /* Any machine-dependent fields. */
struct callout p_itcallout; /* (h + c) Interval timer callout. */
u_short p_acflag; /* (c) Accounting flags. */
struct proc *p_peers; /* (r) */
struct proc *p_leader; /* (b) */
void *p_emuldata; /* (c) Emulator state data. */
struct label *p_label; /* (*) Proc (not subject) MAC label. */
STAILQ_HEAD(, ktr_request) p_ktr; /* (o) KTR event queue. */
LIST_HEAD(, mqueue_notifier) p_mqnotifier; /* (c) mqueue notifiers.*/
struct kdtrace_proc *p_dtrace; /* (*) DTrace-specific data. */
struct cv p_pwait; /* (*) wait cv for exit/exec. */
uint64_t p_prev_runtime; /* (c) Resource usage accounting. */
struct racct *p_racct; /* (b) Resource accounting. */
int p_throttled; /* (c) Flag for racct pcpu throttling */
/*
* An orphan is the child that has been re-parented to the
* debugger as a result of attaching to it. Need to keep
* track of them for parent to be able to collect the exit
* status of what used to be children.
*/
LIST_ENTRY(proc) p_orphan; /* (e) List of orphan processes. */
LIST_HEAD(, proc) p_orphans; /* (e) Pointer to list of orphans. */
};
#define p_session p_pgrp->pg_session
#define p_pgid p_pgrp->pg_id
#define NOCPU (-1) /* For when we aren't on a CPU. */
#define NOCPU_OLD (255)
#define MAXCPU_OLD (254)
#define PROC_SLOCK(p) mtx_lock_spin(&(p)->p_slock)
#define PROC_SUNLOCK(p) mtx_unlock_spin(&(p)->p_slock)
#define PROC_SLOCK_ASSERT(p, type) mtx_assert(&(p)->p_slock, (type))
#define PROC_STATLOCK(p) mtx_lock_spin(&(p)->p_statmtx)
#define PROC_STATUNLOCK(p) mtx_unlock_spin(&(p)->p_statmtx)
#define PROC_STATLOCK_ASSERT(p, type) mtx_assert(&(p)->p_statmtx, (type))
#define PROC_ITIMLOCK(p) mtx_lock_spin(&(p)->p_itimmtx)
#define PROC_ITIMUNLOCK(p) mtx_unlock_spin(&(p)->p_itimmtx)
#define PROC_ITIMLOCK_ASSERT(p, type) mtx_assert(&(p)->p_itimmtx, (type))
#define PROC_PROFLOCK(p) mtx_lock_spin(&(p)->p_profmtx)
#define PROC_PROFUNLOCK(p) mtx_unlock_spin(&(p)->p_profmtx)
#define PROC_PROFLOCK_ASSERT(p, type) mtx_assert(&(p)->p_profmtx, (type))
/* These flags are kept in p_flag. */
#define P_ADVLOCK 0x00001 /* Process may hold a POSIX advisory lock. */
#define P_CONTROLT 0x00002 /* Has a controlling terminal. */
#define P_KPROC 0x00004 /* Kernel process. */
#define P_UNUSED3 0x00008 /* --available-- */
#define P_PPWAIT 0x00010 /* Parent is waiting for child to exec/exit. */
#define P_PROFIL 0x00020 /* Has started profiling. */
#define P_STOPPROF 0x00040 /* Has thread requesting to stop profiling. */
#define P_HADTHREADS 0x00080 /* Has had threads (no cleanup shortcuts) */
#define P_SUGID 0x00100 /* Had set id privileges since last exec. */
#define P_SYSTEM 0x00200 /* System proc: no sigs, stats or swapping. */
#define P_SINGLE_EXIT 0x00400 /* Threads suspending should exit, not wait. */
#define P_TRACED 0x00800 /* Debugged process being traced. */
#define P_WAITED 0x01000 /* Someone is waiting for us. */
#define P_WEXIT 0x02000 /* Working on exiting. */
#define P_EXEC 0x04000 /* Process called exec. */
#define P_WKILLED 0x08000 /* Killed, go to kernel/user boundary ASAP. */
#define P_CONTINUED 0x10000 /* Proc has continued from a stopped state. */
#define P_STOPPED_SIG 0x20000 /* Stopped due to SIGSTOP/SIGTSTP. */
#define P_STOPPED_TRACE 0x40000 /* Stopped because of tracing. */
#define P_STOPPED_SINGLE 0x80000 /* Only 1 thread can continue (not to user). */
#define P_PROTECTED 0x100000 /* Do not kill on memory overcommit. */
#define P_SIGEVENT 0x200000 /* Process pending signals changed. */
#define P_SINGLE_BOUNDARY 0x400000 /* Threads should suspend at user boundary. */
#define P_HWPMC 0x800000 /* Process is using HWPMCs */
#define P_JAILED 0x1000000 /* Process is in jail. */
#define P_TOTAL_STOP 0x2000000 /* Stopped in stop_all_proc. */
#define P_INEXEC 0x4000000 /* Process is in execve(). */
#define P_STATCHILD 0x8000000 /* Child process stopped or exited. */
#define P_INMEM 0x10000000 /* Loaded into memory. */
#define P_SWAPPINGOUT 0x20000000 /* Process is being swapped out. */
#define P_SWAPPINGIN 0x40000000 /* Process is being swapped in. */
#define P_PPTRACE 0x80000000 /* PT_TRACEME by vforked child. */
#define P_STOPPED (P_STOPPED_SIG|P_STOPPED_SINGLE|P_STOPPED_TRACE)
#define P_SHOULDSTOP(p) ((p)->p_flag & P_STOPPED)
#define P_KILLED(p) ((p)->p_flag & P_WKILLED)
/* These flags are kept in p_flag2. */
#define P2_INHERIT_PROTECTED 0x00000001 /* New children get P_PROTECTED. */
#define P2_NOTRACE 0x00000002 /* No ptrace(2) attach or coredumps. */
#define P2_NOTRACE_EXEC 0x00000004 /* Keep P2_NOPTRACE on exec(2). */
#define P2_AST_SU 0x00000008 /* Handles SU ast for kthreads. */
#define P2_PTRACE_FSTP 0x00000010 /* SIGSTOP from PT_ATTACH not yet handled. */
#define P2_TRAPCAP 0x00000020 /* SIGTRAP on ENOTCAPABLE */
#define P2_ASLR_ENABLE 0x00000040 /* Force enable ASLR. */
#define P2_ASLR_DISABLE 0x00000080 /* Force disable ASLR. */
#define P2_ASLR_IGNSTART 0x00000100 /* Enable ASLR to consume sbrk area. */
#define P2_PROTMAX_ENABLE 0x00000200 /* Force enable implied PROT_MAX. */
#define P2_PROTMAX_DISABLE 0x00000400 /* Force disable implied PROT_MAX. */
#define P2_STKGAP_DISABLE 0x00000800 /* Disable stack gap for MAP_STACK */
#define P2_STKGAP_DISABLE_EXEC 0x00001000 /* Stack gap disabled after exec */
/* Flags protected by proctree_lock, kept in p_treeflags. */
#define P_TREE_ORPHANED 0x00000001 /* Reparented, on orphan list */
#define P_TREE_FIRST_ORPHAN 0x00000002 /* First element of orphan
list */
#define P_TREE_REAPER 0x00000004 /* Reaper of subtree */
/*
* These were process status values (p_stat), now they are only used in
* legacy conversion code.
*/
#define SIDL 1 /* Process being created by fork. */
#define SRUN 2 /* Currently runnable. */
#define SSLEEP 3 /* Sleeping on an address. */
#define SSTOP 4 /* Process debugging or suspension. */
#define SZOMB 5 /* Awaiting collection by parent. */
#define SWAIT 6 /* Waiting for interrupt. */
#define SLOCK 7 /* Blocked on a lock. */
#define P_MAGIC 0xbeefface
#ifdef _KERNEL
/* Types and flags for mi_switch(). */
#define SW_TYPE_MASK 0xff /* First 8 bits are switch type */
#define SWT_NONE 0 /* Unspecified switch. */
#define SWT_PREEMPT 1 /* Switching due to preemption. */
#define SWT_OWEPREEMPT 2 /* Switching due to owepreempt. */
#define SWT_TURNSTILE 3 /* Turnstile contention. */
#define SWT_SLEEPQ 4 /* Sleepq wait. */
#define SWT_SLEEPQTIMO 5 /* Sleepq timeout wait. */
#define SWT_RELINQUISH 6 /* yield call. */
#define SWT_NEEDRESCHED 7 /* NEEDRESCHED was set. */
#define SWT_IDLE 8 /* Switching from the idle thread. */
#define SWT_IWAIT 9 /* Waiting for interrupts. */
#define SWT_SUSPEND 10 /* Thread suspended. */
#define SWT_REMOTEPREEMPT 11 /* Remote processor preempted. */
#define SWT_REMOTEWAKEIDLE 12 /* Remote processor preempted idle. */
#define SWT_COUNT 13 /* Number of switch types. */
/* Flags */
#define SW_VOL 0x0100 /* Voluntary switch. */
#define SW_INVOL 0x0200 /* Involuntary switch. */
#define SW_PREEMPT 0x0400 /* The invol switch is a preemption */
/* How values for thread_single(). */
#define SINGLE_NO_EXIT 0
#define SINGLE_EXIT 1
#define SINGLE_BOUNDARY 2
#define SINGLE_ALLPROC 3
#ifdef MALLOC_DECLARE
MALLOC_DECLARE(M_PARGS);
MALLOC_DECLARE(M_PGRP);
MALLOC_DECLARE(M_SESSION);
MALLOC_DECLARE(M_SUBPROC);
#endif
#define FOREACH_PROC_IN_SYSTEM(p) \
LIST_FOREACH((p), &allproc, p_list)
#define FOREACH_THREAD_IN_PROC(p, td) \
TAILQ_FOREACH((td), &(p)->p_threads, td_plist)
#define FIRST_THREAD_IN_PROC(p) TAILQ_FIRST(&(p)->p_threads)
/*
* We use process IDs <= pid_max <= PID_MAX; PID_MAX + 1 must also fit
* in a pid_t, as it is used to represent "no process group".
*/
#define PID_MAX 99999
#define NO_PID 100000
extern pid_t pid_max;
#define SESS_LEADER(p) ((p)->p_session->s_leader == (p))
#define STOPEVENT(p, e, v) do { \
WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, \
"checking stopevent %d", (e)); \
if ((p)->p_stops & (e)) { \
PROC_LOCK(p); \
stopevent((p), (e), (v)); \
PROC_UNLOCK(p); \
} \
} while (0)
#define _STOPEVENT(p, e, v) do { \
PROC_LOCK_ASSERT(p, MA_OWNED); \
WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, &p->p_mtx.lock_object, \
"checking stopevent %d", (e)); \
if ((p)->p_stops & (e)) \
stopevent((p), (e), (v)); \
} while (0)
/* Lock and unlock a process. */
#define PROC_LOCK(p) mtx_lock(&(p)->p_mtx)
#define PROC_TRYLOCK(p) mtx_trylock(&(p)->p_mtx)
#define PROC_UNLOCK(p) mtx_unlock(&(p)->p_mtx)
#define PROC_LOCKED(p) mtx_owned(&(p)->p_mtx)
#define PROC_LOCK_ASSERT(p, type) mtx_assert(&(p)->p_mtx, (type))
/* Lock and unlock a process group. */
#define PGRP_LOCK(pg) mtx_lock(&(pg)->pg_mtx)
#define PGRP_UNLOCK(pg) mtx_unlock(&(pg)->pg_mtx)
#define PGRP_LOCKED(pg) mtx_owned(&(pg)->pg_mtx)
#define PGRP_LOCK_ASSERT(pg, type) mtx_assert(&(pg)->pg_mtx, (type))
#define PGRP_LOCK_PGSIGNAL(pg) do { \
if ((pg) != NULL) \
PGRP_LOCK(pg); \
} while (0)
#define PGRP_UNLOCK_PGSIGNAL(pg) do { \
if ((pg) != NULL) \
PGRP_UNLOCK(pg); \
} while (0)
/* Lock and unlock a session. */
#define SESS_LOCK(s) mtx_lock(&(s)->s_mtx)
#define SESS_UNLOCK(s) mtx_unlock(&(s)->s_mtx)
#define SESS_LOCKED(s) mtx_owned(&(s)->s_mtx)
#define SESS_LOCK_ASSERT(s, type) mtx_assert(&(s)->s_mtx, (type))
/*
* Non-zero p_lock ensures that:
* - exit1() is not performed until p_lock reaches zero;
* - the process' threads stack are not swapped out if they are currently
* not (P_INMEM).
*
* PHOLD() asserts that the process (except the current process) is
* not exiting, increments p_lock and swaps threads stacks into memory,
* if needed.
* _PHOLD() is same as PHOLD(), it takes the process locked.
* _PHOLD_LITE() also takes the process locked, but comparing with
* _PHOLD(), it only guarantees that exit1() is not executed,
* faultin() is not called.
*/
#define PHOLD(p) do { \
PROC_LOCK(p); \
_PHOLD(p); \
PROC_UNLOCK(p); \
} while (0)
#define _PHOLD(p) do { \
PROC_LOCK_ASSERT((p), MA_OWNED); \
KASSERT(!((p)->p_flag & P_WEXIT) || (p) == curproc, \
("PHOLD of exiting process %p", p)); \
(p)->p_lock++; \
if (((p)->p_flag & P_INMEM) == 0) \
faultin((p)); \
} while (0)
#define _PHOLD_LITE(p) do { \
PROC_LOCK_ASSERT((p), MA_OWNED); \
KASSERT(!((p)->p_flag & P_WEXIT) || (p) == curproc, \
("PHOLD of exiting process %p", p)); \
(p)->p_lock++; \
} while (0)
#define PROC_ASSERT_HELD(p) do { \
KASSERT((p)->p_lock > 0, ("process %p not held", p)); \
} while (0)
#define PRELE(p) do { \
PROC_LOCK((p)); \
_PRELE((p)); \
PROC_UNLOCK((p)); \
} while (0)
#define _PRELE(p) do { \
PROC_LOCK_ASSERT((p), MA_OWNED); \
PROC_ASSERT_HELD(p); \
(--(p)->p_lock); \
if (((p)->p_flag & P_WEXIT) && (p)->p_lock == 0) \
wakeup(&(p)->p_lock); \
} while (0)
#define PROC_ASSERT_NOT_HELD(p) do { \
KASSERT((p)->p_lock == 0, ("process %p held", p)); \
} while (0)
#define PROC_UPDATE_COW(p) do { \
PROC_LOCK_ASSERT((p), MA_OWNED); \
(p)->p_cowgen++; \
} while (0)
/* Check whether a thread is safe to be swapped out. */
#define thread_safetoswapout(td) ((td)->td_flags & TDF_CANSWAP)
/* Control whether or not it is safe for curthread to sleep. */
#define THREAD_NO_SLEEPING() do { \
curthread->td_no_sleeping++; \
MPASS(curthread->td_no_sleeping > 0); \
} while (0)
#define THREAD_SLEEPING_OK() do { \
MPASS(curthread->td_no_sleeping > 0); \
curthread->td_no_sleeping--; \
} while (0)
#define THREAD_CAN_SLEEP() ((curthread)->td_no_sleeping == 0)
#define PIDHASH(pid) (&pidhashtbl[(pid) & pidhash])
#define PIDHASHLOCK(pid) (&pidhashtbl_lock[((pid) & pidhashlock)])
extern LIST_HEAD(pidhashhead, proc) *pidhashtbl;
extern struct sx *pidhashtbl_lock;
extern u_long pidhash;
extern u_long pidhashlock;
#define TIDHASH(tid) (&tidhashtbl[(tid) & tidhash])
extern LIST_HEAD(tidhashhead, thread) *tidhashtbl;
extern u_long tidhash;
extern struct rwlock tidhash_lock;
#define PGRPHASH(pgid) (&pgrphashtbl[(pgid) & pgrphash])
extern LIST_HEAD(pgrphashhead, pgrp) *pgrphashtbl;
extern u_long pgrphash;
extern struct sx allproc_lock;
extern int allproc_gen;
extern struct sx proctree_lock;
extern struct mtx ppeers_lock;
extern struct mtx procid_lock;
extern struct proc proc0; /* Process slot for swapper. */
extern struct thread0_storage thread0_st; /* Primary thread in proc0. */
#define thread0 (thread0_st.t0st_thread)
extern struct vmspace vmspace0; /* VM space for proc0. */
extern int hogticks; /* Limit on kernel cpu hogs. */
extern int lastpid;
extern int nprocs, maxproc; /* Current and max number of procs. */
extern int maxprocperuid; /* Max procs per uid. */
extern u_long ps_arg_cache_limit;
LIST_HEAD(proclist, proc);
TAILQ_HEAD(procqueue, proc);
TAILQ_HEAD(threadqueue, thread);
extern struct proclist allproc; /* List of all processes. */
extern struct proc *initproc, *pageproc; /* Process slots for init, pager. */
extern struct uma_zone *proc_zone;
struct proc *pfind(pid_t); /* Find process by id. */
struct proc *pfind_any(pid_t); /* Find (zombie) process by id. */
struct proc *pfind_any_locked(pid_t pid); /* Find process by id, locked. */
struct pgrp *pgfind(pid_t); /* Find process group by id. */
void pidhash_slockall(void); /* Shared lock all pid hash lists. */
void pidhash_sunlockall(void); /* Shared unlock all pid hash lists. */
struct fork_req {
int fr_flags;
int fr_pages;
int *fr_pidp;
struct proc **fr_procp;
int *fr_pd_fd;
int fr_pd_flags;
struct filecaps *fr_pd_fcaps;
int fr_flags2;
#define FR2_DROPSIG_CAUGHT 0x00001 /* Drop caught non-DFL signals */
};
/*
* pget() flags.
*/
#define PGET_HOLD 0x00001 /* Hold the process. */
#define PGET_CANSEE 0x00002 /* Check against p_cansee(). */
#define PGET_CANDEBUG 0x00004 /* Check against p_candebug(). */
#define PGET_ISCURRENT 0x00008 /* Check that the found process is current. */
#define PGET_NOTWEXIT 0x00010 /* Check that the process is not in P_WEXIT. */
#define PGET_NOTINEXEC 0x00020 /* Check that the process is not in P_INEXEC. */
#define PGET_NOTID 0x00040 /* Do not assume tid if pid > PID_MAX. */
#define PGET_WANTREAD (PGET_HOLD | PGET_CANDEBUG | PGET_NOTWEXIT)
int pget(pid_t pid, int flags, struct proc **pp);
void ast(struct trapframe *framep);
struct thread *choosethread(void);
int cr_cansee(struct ucred *u1, struct ucred *u2);
int cr_canseesocket(struct ucred *cred, struct socket *so);
int cr_canseeothergids(struct ucred *u1, struct ucred *u2);
int cr_canseeotheruids(struct ucred *u1, struct ucred *u2);
int cr_canseejailproc(struct ucred *u1, struct ucred *u2);
int cr_cansignal(struct ucred *cred, struct proc *proc, int signum);
int enterpgrp(struct proc *p, pid_t pgid, struct pgrp *pgrp,
struct session *sess);
int enterthispgrp(struct proc *p, struct pgrp *pgrp);
void faultin(struct proc *p);
void fixjobc(struct proc *p, struct pgrp *pgrp, int entering);
int fork1(struct thread *, struct fork_req *);
void fork_rfppwait(struct thread *);
void fork_exit(void (*)(void *, struct trapframe *), void *,
struct trapframe *);
void fork_return(struct thread *, struct trapframe *);
int inferior(struct proc *p);
void kern_proc_vmmap_resident(struct vm_map *map, struct vm_map_entry *entry,
int *resident_count, bool *super);
void kern_yield(int);
void kick_proc0(void);
void killjobc(void);
int leavepgrp(struct proc *p);
int maybe_preempt(struct thread *td);
void maybe_yield(void);
void mi_switch(int flags);
int p_candebug(struct thread *td, struct proc *p);
int p_cansee(struct thread *td, struct proc *p);
int p_cansched(struct thread *td, struct proc *p);
int p_cansignal(struct thread *td, struct proc *p, int signum);
int p_canwait(struct thread *td, struct proc *p);
struct pargs *pargs_alloc(int len);
void pargs_drop(struct pargs *pa);
void pargs_hold(struct pargs *pa);
int proc_getargv(struct thread *td, struct proc *p, struct sbuf *sb);
int proc_getauxv(struct thread *td, struct proc *p, struct sbuf *sb);
int proc_getenvv(struct thread *td, struct proc *p, struct sbuf *sb);
void procinit(void);
int proc_iterate(int (*cb)(struct proc *, void *), void *cbarg);
void proc_linkup0(struct proc *p, struct thread *td);
void proc_linkup(struct proc *p, struct thread *td);
struct proc *proc_realparent(struct proc *child);
void proc_reap(struct thread *td, struct proc *p, int *status, int options);
void proc_reparent(struct proc *child, struct proc *newparent, bool set_oppid);
void proc_add_orphan(struct proc *child, struct proc *parent);
void proc_set_traced(struct proc *p, bool stop);
void proc_wkilled(struct proc *p);
struct pstats *pstats_alloc(void);
void pstats_fork(struct pstats *src, struct pstats *dst);
void pstats_free(struct pstats *ps);
void proc_clear_orphan(struct proc *p);
void reaper_abandon_children(struct proc *p, bool exiting);
int securelevel_ge(struct ucred *cr, int level);
int securelevel_gt(struct ucred *cr, int level);
void sess_hold(struct session *);
void sess_release(struct session *);
int setrunnable(struct thread *, int);
void setsugid(struct proc *p);
int should_yield(void);
int sigonstack(size_t sp);
void stopevent(struct proc *, u_int, u_int);
struct thread *tdfind(lwpid_t, pid_t);
void threadinit(void);
void tidhash_add(struct thread *);
void tidhash_remove(struct thread *);
void cpu_idle(int);
int cpu_idle_wakeup(int);
extern void (*cpu_idle_hook)(sbintime_t); /* Hook to machdep CPU idler. */
void cpu_switch(struct thread *, struct thread *, struct mtx *);
void cpu_throw(struct thread *, struct thread *) __dead2;
void unsleep(struct thread *);
void userret(struct thread *, struct trapframe *);
void cpu_exit(struct thread *);
void exit1(struct thread *, int, int) __dead2;
void cpu_copy_thread(struct thread *td, struct thread *td0);
bool cpu_exec_vmspace_reuse(struct proc *p, struct vm_map *map);
int cpu_fetch_syscall_args(struct thread *td);
void cpu_fork(struct thread *, struct proc *, struct thread *, int);
void cpu_fork_kthread_handler(struct thread *, void (*)(void *), void *);
int cpu_procctl(struct thread *td, int idtype, id_t id, int com,
void *data);
void cpu_set_syscall_retval(struct thread *, int);
void cpu_set_upcall(struct thread *, void (*)(void *), void *,
stack_t *);
int cpu_set_user_tls(struct thread *, void *tls_base);
void cpu_thread_alloc(struct thread *);
void cpu_thread_clean(struct thread *);
void cpu_thread_exit(struct thread *);
void cpu_thread_free(struct thread *);
void cpu_thread_swapin(struct thread *);
void cpu_thread_swapout(struct thread *);
struct thread *thread_alloc(int pages);
int thread_alloc_stack(struct thread *, int pages);
void thread_cow_get_proc(struct thread *newtd, struct proc *p);
void thread_cow_get(struct thread *newtd, struct thread *td);
void thread_cow_free(struct thread *td);
void thread_cow_update(struct thread *td);
int thread_create(struct thread *td, struct rtprio *rtp,
int (*initialize_thread)(struct thread *, void *), void *thunk);
void thread_exit(void) __dead2;
void thread_free(struct thread *td);
void thread_link(struct thread *td, struct proc *p);
void thread_reap(void);
int thread_single(struct proc *p, int how);
void thread_single_end(struct proc *p, int how);
void thread_stash(struct thread *td);
void thread_stopped(struct proc *p);
void childproc_stopped(struct proc *child, int reason);
void childproc_continued(struct proc *child);
void childproc_exited(struct proc *child);
int thread_suspend_check(int how);
bool thread_suspend_check_needed(void);
void thread_suspend_switch(struct thread *, struct proc *p);
void thread_suspend_one(struct thread *td);
void thread_unlink(struct thread *td);
void thread_unsuspend(struct proc *p);
void thread_wait(struct proc *p);
struct thread *thread_find(struct proc *p, lwpid_t tid);
void stop_all_proc(void);
void resume_all_proc(void);
static __inline int
curthread_pflags_set(int flags)
{
struct thread *td;
int save;
td = curthread;
save = ~flags | (td->td_pflags & flags);
td->td_pflags |= flags;
return (save);
}
static __inline void
curthread_pflags_restore(int save)
{
curthread->td_pflags &= save;
}
static __inline __pure2 struct td_sched *
td_get_sched(struct thread *td)
{
return ((struct td_sched *)&td[1]);
}
extern void (*softdep_ast_cleanup)(struct thread *);
static __inline void
td_softdep_cleanup(struct thread *td)
{
if (td->td_su != NULL && softdep_ast_cleanup != NULL)
softdep_ast_cleanup(td);
}
#define PROC_ID_PID 0
#define PROC_ID_GROUP 1
#define PROC_ID_SESSION 2
#define PROC_ID_REAP 3
void proc_id_set(int type, pid_t id);
void proc_id_set_cond(int type, pid_t id);
void proc_id_clear(int type, pid_t id);
EVENTHANDLER_LIST_DECLARE(process_ctor);
EVENTHANDLER_LIST_DECLARE(process_dtor);
EVENTHANDLER_LIST_DECLARE(process_init);
EVENTHANDLER_LIST_DECLARE(process_fini);
EVENTHANDLER_LIST_DECLARE(process_exit);
EVENTHANDLER_LIST_DECLARE(process_fork);
EVENTHANDLER_LIST_DECLARE(process_exec);
EVENTHANDLER_LIST_DECLARE(thread_ctor);
EVENTHANDLER_LIST_DECLARE(thread_dtor);
EVENTHANDLER_LIST_DECLARE(thread_init);
#endif /* _KERNEL */
#endif /* !_SYS_PROC_H_ */
Index: head/sys/sys/sleepqueue.h
===================================================================
--- head/sys/sys/sleepqueue.h (revision 356056)
+++ head/sys/sys/sleepqueue.h (revision 356057)
@@ -1,122 +1,122 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2004 John Baldwin <jhb@FreeBSD.org>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#ifndef _SYS_SLEEPQUEUE_H_
#define _SYS_SLEEPQUEUE_H_
/*
* Sleep queue interface. Sleep/wakeup, condition variables, and sx
* locks use a sleep queue for the queue of threads blocked on a sleep
* channel.
*
* A thread calls sleepq_lock() to lock the sleep queue chain associated
* with a given wait channel. A thread can then call call sleepq_add() to
* add themself onto a sleep queue and call one of the sleepq_wait()
* functions to actually go to sleep. If a thread needs to abort a sleep
* operation it should call sleepq_release() to unlock the associated sleep
* queue chain lock. If the thread also needs to remove itself from a queue
* it just enqueued itself on, it can use sleepq_remove() instead.
*
* If the thread only wishes to sleep for a limited amount of time, it can
* call sleepq_set_timeout() after sleepq_add() to setup a timeout. It
* should then use one of the sleepq_timedwait() functions to block.
*
* A thread is normally resumed from a sleep queue by either the
* sleepq_signal() or sleepq_broadcast() functions. Sleepq_signal() wakes
* the thread with the highest priority that is sleeping on the specified
* wait channel. Sleepq_broadcast() wakes all threads that are sleeping
* on the specified wait channel. A thread sleeping in an interruptible
* sleep can be interrupted by calling sleepq_abort(). A thread can also
* be removed from a specified sleep queue using the sleepq_remove()
* function. Note that the sleep queue chain must first be locked via
* sleepq_lock() before calling sleepq_abort(), sleepq_broadcast(), or
* sleepq_signal(). These routines each return a boolean that will be true
* if at least one swapped-out thread was resumed. In that case, the caller
* is responsible for waking up the swapper by calling kick_proc0() after
* releasing the sleep queue chain lock.
*
* Each thread allocates a sleep queue at thread creation via sleepq_alloc()
* and releases it at thread destruction via sleepq_free(). Note that
* a sleep queue is not tied to a specific thread and that the sleep queue
* released at thread destruction may not be the same sleep queue that the
* thread allocated when it was created.
*
* XXX: Some other parts of the kernel such as ithread sleeping may end up
* using this interface as well (death to TDI_IWAIT!)
*/
struct lock_object;
struct sleepqueue;
struct thread;
#ifdef _KERNEL
#define SLEEPQ_TYPE 0x0ff /* Mask of sleep queue types. */
#define SLEEPQ_SLEEP 0x00 /* Used by sleep/wakeup. */
#define SLEEPQ_CONDVAR 0x01 /* Used for a cv. */
#define SLEEPQ_PAUSE 0x02 /* Used by pause. */
#define SLEEPQ_SX 0x03 /* Used by an sx lock. */
#define SLEEPQ_LK 0x04 /* Used by a lockmgr. */
#define SLEEPQ_INTERRUPTIBLE 0x100 /* Sleep is interruptible. */
#define SLEEPQ_UNFAIR 0x200 /* Unfair wakeup order. */
void init_sleepqueues(void);
int sleepq_abort(struct thread *td, int intrval);
-void sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg,
- int flags, int queue);
+void sleepq_add(const void *wchan, struct lock_object *lock,
+ const char *wmesg, int flags, int queue);
struct sleepqueue *sleepq_alloc(void);
-int sleepq_broadcast(void *wchan, int flags, int pri, int queue);
+int sleepq_broadcast(const void *wchan, int flags, int pri, int queue);
void sleepq_chains_remove_matching(bool (*matches)(struct thread *));
void sleepq_free(struct sleepqueue *sq);
-void sleepq_lock(void *wchan);
-struct sleepqueue *sleepq_lookup(void *wchan);
-void sleepq_release(void *wchan);
-void sleepq_remove(struct thread *td, void *wchan);
+void sleepq_lock(const void *wchan);
+struct sleepqueue *sleepq_lookup(const void *wchan);
+void sleepq_release(const void *wchan);
+void sleepq_remove(struct thread *td, const void *wchan);
int sleepq_remove_matching(struct sleepqueue *sq, int queue,
bool (*matches)(struct thread *), int pri);
-int sleepq_signal(void *wchan, int flags, int pri, int queue);
-void sleepq_set_timeout_sbt(void *wchan, sbintime_t sbt,
+int sleepq_signal(const void *wchan, int flags, int pri, int queue);
+void sleepq_set_timeout_sbt(const void *wchan, sbintime_t sbt,
sbintime_t pr, int flags);
#define sleepq_set_timeout(wchan, timo) \
sleepq_set_timeout_sbt((wchan), tick_sbt * (timo), 0, C_HARDCLOCK)
-u_int sleepq_sleepcnt(void *wchan, int queue);
-int sleepq_timedwait(void *wchan, int pri);
-int sleepq_timedwait_sig(void *wchan, int pri);
-int sleepq_type(void *wchan);
-void sleepq_wait(void *wchan, int pri);
-int sleepq_wait_sig(void *wchan, int pri);
+u_int sleepq_sleepcnt(const void *wchan, int queue);
+int sleepq_timedwait(const void *wchan, int pri);
+int sleepq_timedwait_sig(const void *wchan, int pri);
+int sleepq_type(const void *wchan);
+void sleepq_wait(const void *wchan, int pri);
+int sleepq_wait_sig(const void *wchan, int pri);
#ifdef STACK
struct sbuf;
-int sleepq_sbuf_print_stacks(struct sbuf *sb, void *wchan, int queue,
+int sleepq_sbuf_print_stacks(struct sbuf *sb, const void *wchan, int queue,
int *count_stacks_printed);
#endif
#endif /* _KERNEL */
#endif /* !_SYS_SLEEPQUEUE_H_ */
Index: head/sys/sys/systm.h
===================================================================
--- head/sys/sys/systm.h (revision 356056)
+++ head/sys/sys/systm.h (revision 356057)
@@ -1,605 +1,605 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1982, 1988, 1991, 1993
* The Regents of the University of California. All rights reserved.
* (c) UNIX System Laboratories, Inc.
* All or some portions of this file are derived from material licensed
* to the University of California by American Telephone and Telegraph
* Co. or Unix System Laboratories, Inc. and are reproduced herein with
* the permission of UNIX System Laboratories, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)systm.h 8.7 (Berkeley) 3/29/95
* $FreeBSD$
*/
#ifndef _SYS_SYSTM_H_
#define _SYS_SYSTM_H_
#include <sys/cdefs.h>
#include <machine/atomic.h>
#include <machine/cpufunc.h>
#include <sys/callout.h>
#include <sys/queue.h>
#include <sys/stdint.h> /* for people using printf mainly */
__NULLABILITY_PRAGMA_PUSH
extern int cold; /* nonzero if we are doing a cold boot */
extern int suspend_blocked; /* block suspend due to pending shutdown */
extern int rebooting; /* kern_reboot() has been called. */
extern const char *panicstr; /* panic message */
extern char version[]; /* system version */
extern char compiler_version[]; /* compiler version */
extern char copyright[]; /* system copyright */
extern int kstack_pages; /* number of kernel stack pages */
extern u_long pagesizes[]; /* supported page sizes */
extern long physmem; /* physical memory */
extern long realmem; /* 'real' memory */
extern char *rootdevnames[2]; /* names of possible root devices */
extern int boothowto; /* reboot flags, from console subsystem */
extern int bootverbose; /* nonzero to print verbose messages */
extern int maxusers; /* system tune hint */
extern int ngroups_max; /* max # of supplemental groups */
extern int vm_guest; /* Running as virtual machine guest? */
/*
* Detected virtual machine guest types. The intention is to expand
* and/or add to the VM_GUEST_VM type if specific VM functionality is
* ever implemented (e.g. vendor-specific paravirtualization features).
* Keep in sync with vm_guest_sysctl_names[].
*/
enum VM_GUEST { VM_GUEST_NO = 0, VM_GUEST_VM, VM_GUEST_XEN, VM_GUEST_HV,
VM_GUEST_VMWARE, VM_GUEST_KVM, VM_GUEST_BHYVE, VM_GUEST_VBOX,
VM_GUEST_PARALLELS, VM_LAST };
/*
* These functions need to be declared before the KASSERT macro is invoked in
* !KASSERT_PANIC_OPTIONAL builds, so their declarations are sort of out of
* place compared to other function definitions in this header. On the other
* hand, this header is a bit disorganized anyway.
*/
void panic(const char *, ...) __dead2 __printflike(1, 2);
void vpanic(const char *, __va_list) __dead2 __printflike(1, 0);
#if defined(WITNESS) || defined(INVARIANT_SUPPORT)
#ifdef KASSERT_PANIC_OPTIONAL
void kassert_panic(const char *fmt, ...) __printflike(1, 2);
#else
#define kassert_panic panic
#endif
#endif
#ifdef INVARIANTS /* The option is always available */
#define KASSERT(exp,msg) do { \
if (__predict_false(!(exp))) \
kassert_panic msg; \
} while (0)
#define VNASSERT(exp, vp, msg) do { \
if (__predict_false(!(exp))) { \
vn_printf(vp, "VNASSERT failed\n"); \
kassert_panic msg; \
} \
} while (0)
#else
#define KASSERT(exp,msg) do { \
} while (0)
#define VNASSERT(exp, vp, msg) do { \
} while (0)
#endif
#ifndef CTASSERT /* Allow lint to override */
#define CTASSERT(x) _Static_assert(x, "compile-time assertion failed")
#endif
#if defined(_KERNEL)
#include <sys/param.h> /* MAXCPU */
#include <sys/pcpu.h> /* curthread */
#include <sys/kpilite.h>
#endif
/*
* Assert that a pointer can be loaded from memory atomically.
*
* This assertion enforces stronger alignment than necessary. For example,
* on some architectures, atomicity for unaligned loads will depend on
* whether or not the load spans multiple cache lines.
*/
#define ASSERT_ATOMIC_LOAD_PTR(var, msg) \
KASSERT(sizeof(var) == sizeof(void *) && \
((uintptr_t)&(var) & (sizeof(void *) - 1)) == 0, msg)
/*
* Assert that a thread is in critical(9) section.
*/
#define CRITICAL_ASSERT(td) \
KASSERT((td)->td_critnest >= 1, ("Not in critical section"));
/*
* If we have already panic'd and this is the thread that called
* panic(), then don't block on any mutexes but silently succeed.
* Otherwise, the kernel will deadlock since the scheduler isn't
* going to run the thread that holds any lock we need.
*/
#define SCHEDULER_STOPPED_TD(td) ({ \
MPASS((td) == curthread); \
__predict_false((td)->td_stopsched); \
})
#define SCHEDULER_STOPPED() SCHEDULER_STOPPED_TD(curthread)
/*
* Align variables.
*/
#define __read_mostly __section(".data.read_mostly")
#define __read_frequently __section(".data.read_frequently")
#define __exclusive_cache_line __aligned(CACHE_LINE_SIZE) \
__section(".data.exclusive_cache_line")
/*
* XXX the hints declarations are even more misplaced than most declarations
* in this file, since they are needed in one file (per arch) and only used
* in two files.
* XXX most of these variables should be const.
*/
extern int osreldate;
extern bool dynamic_kenv;
extern struct mtx kenv_lock;
extern char *kern_envp;
extern char *md_envp;
extern char static_env[];
extern char static_hints[]; /* by config for now */
extern char **kenvp;
extern const void *zero_region; /* address space maps to a zeroed page */
extern int unmapped_buf_allowed;
#ifdef __LP64__
#define IOSIZE_MAX iosize_max()
#define DEVFS_IOSIZE_MAX devfs_iosize_max()
#else
#define IOSIZE_MAX SSIZE_MAX
#define DEVFS_IOSIZE_MAX SSIZE_MAX
#endif
/*
* General function declarations.
*/
struct inpcb;
struct lock_object;
struct malloc_type;
struct mtx;
struct proc;
struct socket;
struct thread;
struct tty;
struct ucred;
struct uio;
struct _jmp_buf;
struct trapframe;
struct eventtimer;
int setjmp(struct _jmp_buf *) __returns_twice;
void longjmp(struct _jmp_buf *, int) __dead2;
int dumpstatus(vm_offset_t addr, off_t count);
int nullop(void);
int eopnotsupp(void);
int ureadc(int, struct uio *);
void hashdestroy(void *, struct malloc_type *, u_long);
void *hashinit(int count, struct malloc_type *type, u_long *hashmask);
void *hashinit_flags(int count, struct malloc_type *type,
u_long *hashmask, int flags);
#define HASH_NOWAIT 0x00000001
#define HASH_WAITOK 0x00000002
void *phashinit(int count, struct malloc_type *type, u_long *nentries);
void *phashinit_flags(int count, struct malloc_type *type, u_long *nentries,
int flags);
void g_waitidle(void);
void cpu_boot(int);
void cpu_flush_dcache(void *, size_t);
void cpu_rootconf(void);
void critical_enter_KBI(void);
void critical_exit_KBI(void);
void critical_exit_preempt(void);
void init_param1(void);
void init_param2(long physpages);
void init_static_kenv(char *, size_t);
void tablefull(const char *);
/*
* Allocate per-thread "current" state in the linuxkpi
*/
extern int (*lkpi_alloc_current)(struct thread *, int);
int linux_alloc_current_noop(struct thread *, int);
#if defined(KLD_MODULE) || defined(KTR_CRITICAL) || !defined(_KERNEL) || defined(GENOFFSET)
#define critical_enter() critical_enter_KBI()
#define critical_exit() critical_exit_KBI()
#else
static __inline void
critical_enter(void)
{
struct thread_lite *td;
td = (struct thread_lite *)curthread;
td->td_critnest++;
__compiler_membar();
}
static __inline void
critical_exit(void)
{
struct thread_lite *td;
td = (struct thread_lite *)curthread;
KASSERT(td->td_critnest != 0,
("critical_exit: td_critnest == 0"));
__compiler_membar();
td->td_critnest--;
__compiler_membar();
if (__predict_false(td->td_owepreempt))
critical_exit_preempt();
}
#endif
#ifdef EARLY_PRINTF
typedef void early_putc_t(int ch);
extern early_putc_t *early_putc;
#endif
int kvprintf(char const *, void (*)(int, void*), void *, int,
__va_list) __printflike(1, 0);
void log(int, const char *, ...) __printflike(2, 3);
void log_console(struct uio *);
void vlog(int, const char *, __va_list) __printflike(2, 0);
int asprintf(char **ret, struct malloc_type *mtp, const char *format,
...) __printflike(3, 4);
int printf(const char *, ...) __printflike(1, 2);
int snprintf(char *, size_t, const char *, ...) __printflike(3, 4);
int sprintf(char *buf, const char *, ...) __printflike(2, 3);
int uprintf(const char *, ...) __printflike(1, 2);
int vprintf(const char *, __va_list) __printflike(1, 0);
int vasprintf(char **ret, struct malloc_type *mtp, const char *format,
__va_list ap) __printflike(3, 0);
int vsnprintf(char *, size_t, const char *, __va_list) __printflike(3, 0);
int vsnrprintf(char *, size_t, int, const char *, __va_list) __printflike(4, 0);
int vsprintf(char *buf, const char *, __va_list) __printflike(2, 0);
int sscanf(const char *, char const * _Nonnull, ...) __scanflike(2, 3);
int vsscanf(const char * _Nonnull, char const * _Nonnull, __va_list) __scanflike(2, 0);
long strtol(const char *, char **, int);
u_long strtoul(const char *, char **, int);
quad_t strtoq(const char *, char **, int);
u_quad_t strtouq(const char *, char **, int);
void tprintf(struct proc *p, int pri, const char *, ...) __printflike(3, 4);
void vtprintf(struct proc *, int, const char *, __va_list) __printflike(3, 0);
void hexdump(const void *ptr, int length, const char *hdr, int flags);
#define HD_COLUMN_MASK 0xff
#define HD_DELIM_MASK 0xff00
#define HD_OMIT_COUNT (1 << 16)
#define HD_OMIT_HEX (1 << 17)
#define HD_OMIT_CHARS (1 << 18)
#define ovbcopy(f, t, l) bcopy((f), (t), (l))
void bcopy(const void * _Nonnull from, void * _Nonnull to, size_t len);
void bzero(void * _Nonnull buf, size_t len);
void explicit_bzero(void * _Nonnull, size_t);
int bcmp(const void *b1, const void *b2, size_t len);
void *memset(void * _Nonnull buf, int c, size_t len);
void *memcpy(void * _Nonnull to, const void * _Nonnull from, size_t len);
void *memmove(void * _Nonnull dest, const void * _Nonnull src, size_t n);
int memcmp(const void *b1, const void *b2, size_t len);
#ifdef KCSAN
void *kcsan_memset(void *, int, size_t);
void *kcsan_memcpy(void *, const void *, size_t);
void *kcsan_memmove(void *, const void *, size_t);
int kcsan_memcmp(const void *, const void *, size_t);
#define bcopy(from, to, len) kcsan_memmove((to), (from), (len))
#define bzero(buf, len) kcsan_memset((buf), 0, (len))
#define bcmp(b1, b2, len) kcsan_memcmp((b1), (b2), (len))
#define memset(buf, c, len) kcsan_memset((buf), (c), (len))
#define memcpy(to, from, len) kcsan_memcpy((to), (from), (len))
#define memmove(dest, src, n) kcsan_memmove((dest), (src), (n))
#define memcmp(b1, b2, len) kcsan_memcmp((b1), (b2), (len))
#else
#define bcopy(from, to, len) __builtin_memmove((to), (from), (len))
#define bzero(buf, len) __builtin_memset((buf), 0, (len))
#define bcmp(b1, b2, len) __builtin_memcmp((b1), (b2), (len))
#define memset(buf, c, len) __builtin_memset((buf), (c), (len))
#define memcpy(to, from, len) __builtin_memcpy((to), (from), (len))
#define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
#define memcmp(b1, b2, len) __builtin_memcmp((b1), (b2), (len))
#endif
void *memset_early(void * _Nonnull buf, int c, size_t len);
#define bzero_early(buf, len) memset_early((buf), 0, (len))
void *memcpy_early(void * _Nonnull to, const void * _Nonnull from, size_t len);
void *memmove_early(void * _Nonnull dest, const void * _Nonnull src, size_t n);
#define bcopy_early(from, to, len) memmove_early((to), (from), (len))
int copystr(const void * _Nonnull __restrict kfaddr,
void * _Nonnull __restrict kdaddr, size_t len,
size_t * __restrict lencopied);
int copyinstr(const void * __restrict udaddr,
void * _Nonnull __restrict kaddr, size_t len,
size_t * __restrict lencopied);
int copyin(const void * __restrict udaddr,
void * _Nonnull __restrict kaddr, size_t len);
int copyin_nofault(const void * __restrict udaddr,
void * _Nonnull __restrict kaddr, size_t len);
int copyout(const void * _Nonnull __restrict kaddr,
void * __restrict udaddr, size_t len);
int copyout_nofault(const void * _Nonnull __restrict kaddr,
void * __restrict udaddr, size_t len);
#ifdef KCSAN
int kcsan_copystr(const void *, void *, size_t, size_t *);
int kcsan_copyin(const void *, void *, size_t);
int kcsan_copyinstr(const void *, void *, size_t, size_t *);
int kcsan_copyout(const void *, void *, size_t);
#define copystr(kf, k, l, lc) kcsan_copystr((kf), (k), (l), (lc))
#define copyin(u, k, l) kcsan_copyin((u), (k), (l))
#define copyinstr(u, k, l, lc) kcsan_copyinstr((u), (k), (l), (lc))
#define copyout(k, u, l) kcsan_copyout((k), (u), (l))
#endif
int fubyte(volatile const void *base);
long fuword(volatile const void *base);
int fuword16(volatile const void *base);
int32_t fuword32(volatile const void *base);
int64_t fuword64(volatile const void *base);
int fueword(volatile const void *base, long *val);
int fueword32(volatile const void *base, int32_t *val);
int fueword64(volatile const void *base, int64_t *val);
int subyte(volatile void *base, int byte);
int suword(volatile void *base, long word);
int suword16(volatile void *base, int word);
int suword32(volatile void *base, int32_t word);
int suword64(volatile void *base, int64_t word);
uint32_t casuword32(volatile uint32_t *base, uint32_t oldval, uint32_t newval);
u_long casuword(volatile u_long *p, u_long oldval, u_long newval);
int casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
uint32_t newval);
int casueword(volatile u_long *p, u_long oldval, u_long *oldvalp,
u_long newval);
void realitexpire(void *);
int sysbeep(int hertz, int period);
void hardclock(int cnt, int usermode);
void hardclock_sync(int cpu);
void softclock(void *);
void statclock(int cnt, int usermode);
void profclock(int cnt, int usermode, uintfptr_t pc);
int hardclockintr(void);
void startprofclock(struct proc *);
void stopprofclock(struct proc *);
void cpu_startprofclock(void);
void cpu_stopprofclock(void);
void suspendclock(void);
void resumeclock(void);
sbintime_t cpu_idleclock(void);
void cpu_activeclock(void);
void cpu_new_callout(int cpu, sbintime_t bt, sbintime_t bt_opt);
void cpu_et_frequency(struct eventtimer *et, uint64_t newfreq);
extern int cpu_disable_c2_sleep;
extern int cpu_disable_c3_sleep;
char *kern_getenv(const char *name);
void freeenv(char *env);
int getenv_int(const char *name, int *data);
int getenv_uint(const char *name, unsigned int *data);
int getenv_long(const char *name, long *data);
int getenv_ulong(const char *name, unsigned long *data);
int getenv_string(const char *name, char *data, int size);
int getenv_int64(const char *name, int64_t *data);
int getenv_uint64(const char *name, uint64_t *data);
int getenv_quad(const char *name, quad_t *data);
int kern_setenv(const char *name, const char *value);
int kern_unsetenv(const char *name);
int testenv(const char *name);
int getenv_array(const char *name, void *data, int size, int *psize,
int type_size, bool allow_signed);
#define GETENV_UNSIGNED false /* negative numbers not allowed */
#define GETENV_SIGNED true /* negative numbers allowed */
typedef uint64_t (cpu_tick_f)(void);
void set_cputicker(cpu_tick_f *func, uint64_t freq, unsigned var);
extern cpu_tick_f *cpu_ticks;
uint64_t cpu_tickrate(void);
uint64_t cputick2usec(uint64_t tick);
#ifdef APM_FIXUP_CALLTODO
struct timeval;
void adjust_timeout_calltodo(struct timeval *time_change);
#endif /* APM_FIXUP_CALLTODO */
#include <sys/libkern.h>
/* Initialize the world */
void consinit(void);
void cpu_initclocks(void);
void cpu_initclocks_bsp(void);
void cpu_initclocks_ap(void);
void usrinfoinit(void);
/* Finalize the world */
void kern_reboot(int) __dead2;
void shutdown_nice(int);
/* Stubs for obsolete functions that used to be for interrupt management */
static __inline intrmask_t splbio(void) { return 0; }
static __inline intrmask_t splcam(void) { return 0; }
static __inline intrmask_t splclock(void) { return 0; }
static __inline intrmask_t splhigh(void) { return 0; }
static __inline intrmask_t splimp(void) { return 0; }
static __inline intrmask_t splnet(void) { return 0; }
static __inline intrmask_t spltty(void) { return 0; }
static __inline void splx(intrmask_t ipl __unused) { return; }
/*
* Common `proc' functions are declared here so that proc.h can be included
* less often.
*/
-int _sleep(void * _Nonnull chan, struct lock_object *lock, int pri,
+int _sleep(const void * _Nonnull chan, struct lock_object *lock, int pri,
const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags);
#define msleep(chan, mtx, pri, wmesg, timo) \
_sleep((chan), &(mtx)->lock_object, (pri), (wmesg), \
tick_sbt * (timo), 0, C_HARDCLOCK)
#define msleep_sbt(chan, mtx, pri, wmesg, bt, pr, flags) \
_sleep((chan), &(mtx)->lock_object, (pri), (wmesg), (bt), (pr), \
(flags))
-int msleep_spin_sbt(void * _Nonnull chan, struct mtx *mtx,
+int msleep_spin_sbt(const void * _Nonnull chan, struct mtx *mtx,
const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags);
#define msleep_spin(chan, mtx, wmesg, timo) \
msleep_spin_sbt((chan), (mtx), (wmesg), tick_sbt * (timo), \
0, C_HARDCLOCK)
int pause_sbt(const char *wmesg, sbintime_t sbt, sbintime_t pr,
int flags);
#define pause(wmesg, timo) \
pause_sbt((wmesg), tick_sbt * (timo), 0, C_HARDCLOCK)
#define pause_sig(wmesg, timo) \
pause_sbt((wmesg), tick_sbt * (timo), 0, C_HARDCLOCK | C_CATCH)
#define tsleep(chan, pri, wmesg, timo) \
_sleep((chan), NULL, (pri), (wmesg), tick_sbt * (timo), \
0, C_HARDCLOCK)
#define tsleep_sbt(chan, pri, wmesg, bt, pr, flags) \
_sleep((chan), NULL, (pri), (wmesg), (bt), (pr), (flags))
-void wakeup(void * chan);
-void wakeup_one(void * chan);
-void wakeup_any(void * chan);
+void wakeup(const void *chan);
+void wakeup_one(const void *chan);
+void wakeup_any(const void *chan);
/*
* Common `struct cdev *' stuff are declared here to avoid #include poisoning
*/
struct cdev;
dev_t dev2udev(struct cdev *x);
const char *devtoname(struct cdev *cdev);
#ifdef __LP64__
size_t devfs_iosize_max(void);
size_t iosize_max(void);
#endif
int poll_no_poll(int events);
/* XXX: Should be void nanodelay(u_int nsec); */
void DELAY(int usec);
/* Root mount holdback API */
struct root_hold_token {
int flags;
const char *who;
TAILQ_ENTRY(root_hold_token) list;
};
struct root_hold_token *root_mount_hold(const char *identifier);
void root_mount_hold_token(const char *identifier, struct root_hold_token *h);
void root_mount_rel(struct root_hold_token *h);
int root_mounted(void);
/*
* Unit number allocation API. (kern/subr_unit.c)
*/
struct unrhdr;
struct unrhdr *new_unrhdr(int low, int high, struct mtx *mutex);
void init_unrhdr(struct unrhdr *uh, int low, int high, struct mtx *mutex);
void delete_unrhdr(struct unrhdr *uh);
void clear_unrhdr(struct unrhdr *uh);
void clean_unrhdr(struct unrhdr *uh);
void clean_unrhdrl(struct unrhdr *uh);
int alloc_unr(struct unrhdr *uh);
int alloc_unr_specific(struct unrhdr *uh, u_int item);
int alloc_unrl(struct unrhdr *uh);
void free_unr(struct unrhdr *uh, u_int item);
#ifndef __LP64__
#define UNR64_LOCKED
#endif
struct unrhdr64 {
uint64_t counter;
};
static __inline void
new_unrhdr64(struct unrhdr64 *unr64, uint64_t low)
{
unr64->counter = low;
}
#ifdef UNR64_LOCKED
uint64_t alloc_unr64(struct unrhdr64 *);
#else
static __inline uint64_t
alloc_unr64(struct unrhdr64 *unr64)
{
return (atomic_fetchadd_64(&unr64->counter, 1));
}
#endif
void intr_prof_stack_use(struct thread *td, struct trapframe *frame);
void counted_warning(unsigned *counter, const char *msg);
/*
* APIs to manage deprecation and obsolescence.
*/
struct device;
void _gone_in(int major, const char *msg);
void _gone_in_dev(struct device *dev, int major, const char *msg);
#ifdef NO_OBSOLETE_CODE
#define __gone_ok(m, msg) \
_Static_assert(m < P_OSREL_MAJOR(__FreeBSD_version)), \
"Obsolete code" msg);
#else
#define __gone_ok(m, msg)
#endif
#define gone_in(major, msg) __gone_ok(major, msg) _gone_in(major, msg)
#define gone_in_dev(dev, major, msg) __gone_ok(major, msg) _gone_in_dev(dev, major, msg)
__NULLABILITY_PRAGMA_POP
#endif /* !_SYS_SYSTM_H_ */
Index: head/sys/sys/user.h
===================================================================
--- head/sys/sys/user.h (revision 356056)
+++ head/sys/sys/user.h (revision 356057)
@@ -1,614 +1,614 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1982, 1986, 1989, 1991, 1993
* The Regents of the University of California.
* Copyright (c) 2007 Robert N. M. Watson
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)user.h 8.2 (Berkeley) 9/23/93
* $FreeBSD$
*/
#ifndef _SYS_USER_H_
#define _SYS_USER_H_
#include <machine/pcb.h>
#ifndef _KERNEL
/* stuff that *used* to be included by user.h, or is now needed */
#include <sys/errno.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <sys/ucred.h>
#include <sys/uio.h>
#include <sys/queue.h>
#include <sys/_lock.h>
#include <sys/_mutex.h>
#include <sys/proc.h>
#include <vm/vm.h> /* XXX */
#include <vm/vm_param.h> /* XXX */
#include <vm/pmap.h> /* XXX */
#include <vm/vm_map.h> /* XXX */
#endif /* !_KERNEL */
#ifndef _SYS_RESOURCEVAR_H_
#include <sys/resourcevar.h>
#endif
#ifndef _SYS_SIGNALVAR_H_
#include <sys/signalvar.h>
#endif
#ifndef _SYS_SOCKET_VAR_H_
#include <sys/socket.h>
#endif
#include <sys/caprights.h>
/*
* KERN_PROC subtype ops return arrays of selected proc structure entries:
*
* This struct includes several arrays of spare space, with different arrays
* for different standard C-types. When adding new variables to this struct,
* the space for byte-aligned data should be taken from the ki_sparestring,
* pointers from ki_spareptrs, word-aligned data from ki_spareints, and
* doubleword-aligned data from ki_sparelongs. Make sure the space for new
* variables come from the array which matches the size and alignment of
* those variables on ALL hardware platforms, and then adjust the appropriate
* KI_NSPARE_* value(s) to match.
*
* Always verify that sizeof(struct kinfo_proc) == KINFO_PROC_SIZE on all
* platforms after you have added new variables. Note that if you change
* the value of KINFO_PROC_SIZE, then many userland programs will stop
* working until they are recompiled!
*
* Once you have added the new field, you will need to add code to initialize
* it in two places: function fill_kinfo_proc in sys/kern/kern_proc.c and
* function kvm_proclist in lib/libkvm/kvm_proc.c .
*/
#define KI_NSPARE_INT 2
#define KI_NSPARE_LONG 12
#define KI_NSPARE_PTR 6
#ifndef _KERNEL
#ifndef KINFO_PROC_SIZE
#error "Unknown architecture"
#endif
#endif /* !_KERNEL */
#define WMESGLEN 8 /* size of returned wchan message */
#define LOCKNAMELEN 8 /* size of returned lock name */
#define TDNAMLEN 16 /* size of returned thread name */
#define COMMLEN 19 /* size of returned ki_comm name */
#define KI_EMULNAMELEN 16 /* size of returned ki_emul */
#define KI_NGROUPS 16 /* number of groups in ki_groups */
#define LOGNAMELEN 17 /* size of returned ki_login */
#define LOGINCLASSLEN 17 /* size of returned ki_loginclass */
#ifndef BURN_BRIDGES
#define OCOMMLEN TDNAMLEN
#define ki_ocomm ki_tdname
#endif
/* Flags for the process credential. */
#define KI_CRF_CAPABILITY_MODE 0x00000001
/*
* Steal a bit from ki_cr_flags to indicate that the cred had more than
* KI_NGROUPS groups.
*/
#define KI_CRF_GRP_OVERFLOW 0x80000000
struct kinfo_proc {
int ki_structsize; /* size of this structure */
int ki_layout; /* reserved: layout identifier */
struct pargs *ki_args; /* address of command arguments */
struct proc *ki_paddr; /* address of proc */
struct user *ki_addr; /* kernel virtual addr of u-area */
struct vnode *ki_tracep; /* pointer to trace file */
struct vnode *ki_textvp; /* pointer to executable file */
struct filedesc *ki_fd; /* pointer to open file info */
struct vmspace *ki_vmspace; /* pointer to kernel vmspace struct */
- void *ki_wchan; /* sleep address */
+ const void *ki_wchan; /* sleep address */
pid_t ki_pid; /* Process identifier */
pid_t ki_ppid; /* parent process id */
pid_t ki_pgid; /* process group id */
pid_t ki_tpgid; /* tty process group id */
pid_t ki_sid; /* Process session ID */
pid_t ki_tsid; /* Terminal session ID */
short ki_jobc; /* job control counter */
short ki_spare_short1; /* unused (just here for alignment) */
uint32_t ki_tdev_freebsd11; /* controlling tty dev */
sigset_t ki_siglist; /* Signals arrived but not delivered */
sigset_t ki_sigmask; /* Current signal mask */
sigset_t ki_sigignore; /* Signals being ignored */
sigset_t ki_sigcatch; /* Signals being caught by user */
uid_t ki_uid; /* effective user id */
uid_t ki_ruid; /* Real user id */
uid_t ki_svuid; /* Saved effective user id */
gid_t ki_rgid; /* Real group id */
gid_t ki_svgid; /* Saved effective group id */
short ki_ngroups; /* number of groups */
short ki_spare_short2; /* unused (just here for alignment) */
gid_t ki_groups[KI_NGROUPS]; /* groups */
vm_size_t ki_size; /* virtual size */
segsz_t ki_rssize; /* current resident set size in pages */
segsz_t ki_swrss; /* resident set size before last swap */
segsz_t ki_tsize; /* text size (pages) XXX */
segsz_t ki_dsize; /* data size (pages) XXX */
segsz_t ki_ssize; /* stack size (pages) */
u_short ki_xstat; /* Exit status for wait & stop signal */
u_short ki_acflag; /* Accounting flags */
fixpt_t ki_pctcpu; /* %cpu for process during ki_swtime */
u_int ki_estcpu; /* Time averaged value of ki_cpticks */
u_int ki_slptime; /* Time since last blocked */
u_int ki_swtime; /* Time swapped in or out */
u_int ki_cow; /* number of copy-on-write faults */
u_int64_t ki_runtime; /* Real time in microsec */
struct timeval ki_start; /* starting time */
struct timeval ki_childtime; /* time used by process children */
long ki_flag; /* P_* flags */
long ki_kiflag; /* KI_* flags (below) */
int ki_traceflag; /* Kernel trace points */
char ki_stat; /* S* process status */
signed char ki_nice; /* Process "nice" value */
char ki_lock; /* Process lock (prevent swap) count */
char ki_rqindex; /* Run queue index */
u_char ki_oncpu_old; /* Which cpu we are on (legacy) */
u_char ki_lastcpu_old; /* Last cpu we were on (legacy) */
char ki_tdname[TDNAMLEN+1]; /* thread name */
char ki_wmesg[WMESGLEN+1]; /* wchan message */
char ki_login[LOGNAMELEN+1]; /* setlogin name */
char ki_lockname[LOCKNAMELEN+1]; /* lock name */
char ki_comm[COMMLEN+1]; /* command name */
char ki_emul[KI_EMULNAMELEN+1]; /* emulation name */
char ki_loginclass[LOGINCLASSLEN+1]; /* login class */
char ki_moretdname[MAXCOMLEN-TDNAMLEN+1]; /* more thread name */
/*
* When adding new variables, take space for char-strings from the
* front of ki_sparestrings, and ints from the end of ki_spareints.
* That way the spare room from both arrays will remain contiguous.
*/
char ki_sparestrings[46]; /* spare string space */
int ki_spareints[KI_NSPARE_INT]; /* spare room for growth */
uint64_t ki_tdev; /* controlling tty dev */
int ki_oncpu; /* Which cpu we are on */
int ki_lastcpu; /* Last cpu we were on */
int ki_tracer; /* Pid of tracing process */
int ki_flag2; /* P2_* flags */
int ki_fibnum; /* Default FIB number */
u_int ki_cr_flags; /* Credential flags */
int ki_jid; /* Process jail ID */
int ki_numthreads; /* XXXKSE number of threads in total */
lwpid_t ki_tid; /* XXXKSE thread id */
struct priority ki_pri; /* process priority */
struct rusage ki_rusage; /* process rusage statistics */
/* XXX - most fields in ki_rusage_ch are not (yet) filled in */
struct rusage ki_rusage_ch; /* rusage of children processes */
struct pcb *ki_pcb; /* kernel virtual addr of pcb */
void *ki_kstack; /* kernel virtual addr of stack */
void *ki_udata; /* User convenience pointer */
struct thread *ki_tdaddr; /* address of thread */
/*
* When adding new variables, take space for pointers from the
* front of ki_spareptrs, and longs from the end of ki_sparelongs.
* That way the spare room from both arrays will remain contiguous.
*/
void *ki_spareptrs[KI_NSPARE_PTR]; /* spare room for growth */
long ki_sparelongs[KI_NSPARE_LONG]; /* spare room for growth */
long ki_sflag; /* PS_* flags */
long ki_tdflags; /* XXXKSE kthread flag */
};
void fill_kinfo_proc(struct proc *, struct kinfo_proc *);
/* XXX - the following two defines are temporary */
#define ki_childstime ki_rusage_ch.ru_stime
#define ki_childutime ki_rusage_ch.ru_utime
/*
* Legacy PS_ flag. This moved to p_flag but is maintained for
* compatibility.
*/
#define PS_INMEM 0x00001 /* Loaded into memory. */
/* ki_sessflag values */
#define KI_CTTY 0x00000001 /* controlling tty vnode active */
#define KI_SLEADER 0x00000002 /* session leader */
#define KI_LOCKBLOCK 0x00000004 /* proc blocked on lock ki_lockname */
/*
* This used to be the per-process structure containing data that
* isn't needed in core when the process is swapped out, but now it
* remains only for the benefit of a.out core dumps.
*/
struct user {
struct pstats u_stats; /* *p_stats */
struct kinfo_proc u_kproc; /* eproc */
};
/*
* The KERN_PROC_FILE sysctl allows a process to dump the file descriptor
* array of another process.
*/
#define KF_ATTR_VALID 0x0001
#define KF_TYPE_NONE 0
#define KF_TYPE_VNODE 1
#define KF_TYPE_SOCKET 2
#define KF_TYPE_PIPE 3
#define KF_TYPE_FIFO 4
#define KF_TYPE_KQUEUE 5
#define KF_TYPE_CRYPTO 6
#define KF_TYPE_MQUEUE 7
#define KF_TYPE_SHM 8
#define KF_TYPE_SEM 9
#define KF_TYPE_PTS 10
#define KF_TYPE_PROCDESC 11
#define KF_TYPE_DEV 12
#define KF_TYPE_UNKNOWN 255
#define KF_VTYPE_VNON 0
#define KF_VTYPE_VREG 1
#define KF_VTYPE_VDIR 2
#define KF_VTYPE_VBLK 3
#define KF_VTYPE_VCHR 4
#define KF_VTYPE_VLNK 5
#define KF_VTYPE_VSOCK 6
#define KF_VTYPE_VFIFO 7
#define KF_VTYPE_VBAD 8
#define KF_VTYPE_UNKNOWN 255
#define KF_FD_TYPE_CWD -1 /* Current working directory */
#define KF_FD_TYPE_ROOT -2 /* Root directory */
#define KF_FD_TYPE_JAIL -3 /* Jail directory */
#define KF_FD_TYPE_TRACE -4 /* Ktrace vnode */
#define KF_FD_TYPE_TEXT -5 /* Text vnode */
#define KF_FD_TYPE_CTTY -6 /* Controlling terminal */
#define KF_FLAG_READ 0x00000001
#define KF_FLAG_WRITE 0x00000002
#define KF_FLAG_APPEND 0x00000004
#define KF_FLAG_ASYNC 0x00000008
#define KF_FLAG_FSYNC 0x00000010
#define KF_FLAG_NONBLOCK 0x00000020
#define KF_FLAG_DIRECT 0x00000040
#define KF_FLAG_HASLOCK 0x00000080
#define KF_FLAG_SHLOCK 0x00000100
#define KF_FLAG_EXLOCK 0x00000200
#define KF_FLAG_NOFOLLOW 0x00000400
#define KF_FLAG_CREAT 0x00000800
#define KF_FLAG_TRUNC 0x00001000
#define KF_FLAG_EXCL 0x00002000
#define KF_FLAG_EXEC 0x00004000
/*
* Old format. Has variable hidden padding due to alignment.
* This is a compatibility hack for pre-build 7.1 packages.
*/
#if defined(__amd64__)
#define KINFO_OFILE_SIZE 1328
#endif
#if defined(__i386__)
#define KINFO_OFILE_SIZE 1324
#endif
struct kinfo_ofile {
int kf_structsize; /* Size of kinfo_file. */
int kf_type; /* Descriptor type. */
int kf_fd; /* Array index. */
int kf_ref_count; /* Reference count. */
int kf_flags; /* Flags. */
/* XXX Hidden alignment padding here on amd64 */
off_t kf_offset; /* Seek location. */
int kf_vnode_type; /* Vnode type. */
int kf_sock_domain; /* Socket domain. */
int kf_sock_type; /* Socket type. */
int kf_sock_protocol; /* Socket protocol. */
char kf_path[PATH_MAX]; /* Path to file, if any. */
struct sockaddr_storage kf_sa_local; /* Socket address. */
struct sockaddr_storage kf_sa_peer; /* Peer address. */
};
#if defined(__amd64__) || defined(__i386__)
/*
* This size should never be changed. If you really need to, you must provide
* backward ABI compatibility by allocating a new sysctl MIB that will return
* the new structure. The current structure has to be returned by the current
* sysctl MIB. See how it is done for the kinfo_ofile structure.
*/
#define KINFO_FILE_SIZE 1392
#endif
struct kinfo_file {
int kf_structsize; /* Variable size of record. */
int kf_type; /* Descriptor type. */
int kf_fd; /* Array index. */
int kf_ref_count; /* Reference count. */
int kf_flags; /* Flags. */
int kf_pad0; /* Round to 64 bit alignment. */
int64_t kf_offset; /* Seek location. */
union {
struct {
/* API compatiblity with FreeBSD < 12. */
int kf_vnode_type;
int kf_sock_domain;
int kf_sock_type;
int kf_sock_protocol;
struct sockaddr_storage kf_sa_local;
struct sockaddr_storage kf_sa_peer;
};
union {
struct {
/* Sendq size */
uint32_t kf_sock_sendq;
/* Socket domain. */
int kf_sock_domain0;
/* Socket type. */
int kf_sock_type0;
/* Socket protocol. */
int kf_sock_protocol0;
/* Socket address. */
struct sockaddr_storage kf_sa_local;
/* Peer address. */
struct sockaddr_storage kf_sa_peer;
/* Address of so_pcb. */
uint64_t kf_sock_pcb;
/* Address of inp_ppcb. */
uint64_t kf_sock_inpcb;
/* Address of unp_conn. */
uint64_t kf_sock_unpconn;
/* Send buffer state. */
uint16_t kf_sock_snd_sb_state;
/* Receive buffer state. */
uint16_t kf_sock_rcv_sb_state;
/* Recvq size. */
uint32_t kf_sock_recvq;
} kf_sock;
struct {
/* Vnode type. */
int kf_file_type;
/* Space for future use */
int kf_spareint[3];
uint64_t kf_spareint64[30];
/* Vnode filesystem id. */
uint64_t kf_file_fsid;
/* File device. */
uint64_t kf_file_rdev;
/* Global file id. */
uint64_t kf_file_fileid;
/* File size. */
uint64_t kf_file_size;
/* Vnode filesystem id, FreeBSD 11 compat. */
uint32_t kf_file_fsid_freebsd11;
/* File device, FreeBSD 11 compat. */
uint32_t kf_file_rdev_freebsd11;
/* File mode. */
uint16_t kf_file_mode;
/* Round to 64 bit alignment. */
uint16_t kf_file_pad0;
uint32_t kf_file_pad1;
} kf_file;
struct {
uint32_t kf_spareint[4];
uint64_t kf_spareint64[32];
uint32_t kf_sem_value;
uint16_t kf_sem_mode;
} kf_sem;
struct {
uint32_t kf_spareint[4];
uint64_t kf_spareint64[32];
uint64_t kf_pipe_addr;
uint64_t kf_pipe_peer;
uint32_t kf_pipe_buffer_cnt;
/* Round to 64 bit alignment. */
uint32_t kf_pipe_pad0[3];
} kf_pipe;
struct {
uint32_t kf_spareint[4];
uint64_t kf_spareint64[32];
uint32_t kf_pts_dev_freebsd11;
uint32_t kf_pts_pad0;
uint64_t kf_pts_dev;
/* Round to 64 bit alignment. */
uint32_t kf_pts_pad1[4];
} kf_pts;
struct {
uint32_t kf_spareint[4];
uint64_t kf_spareint64[32];
pid_t kf_pid;
} kf_proc;
} kf_un;
};
uint16_t kf_status; /* Status flags. */
uint16_t kf_pad1; /* Round to 32 bit alignment. */
int _kf_ispare0; /* Space for more stuff. */
cap_rights_t kf_cap_rights; /* Capability rights. */
uint64_t _kf_cap_spare; /* Space for future cap_rights_t. */
/* Truncated before copyout in sysctl */
char kf_path[PATH_MAX]; /* Path to file, if any. */
};
/*
* The KERN_PROC_VMMAP sysctl allows a process to dump the VM layout of
* another process as a series of entries.
*/
#define KVME_TYPE_NONE 0
#define KVME_TYPE_DEFAULT 1
#define KVME_TYPE_VNODE 2
#define KVME_TYPE_SWAP 3
#define KVME_TYPE_DEVICE 4
#define KVME_TYPE_PHYS 5
#define KVME_TYPE_DEAD 6
#define KVME_TYPE_SG 7
#define KVME_TYPE_MGTDEVICE 8
#define KVME_TYPE_UNKNOWN 255
#define KVME_PROT_READ 0x00000001
#define KVME_PROT_WRITE 0x00000002
#define KVME_PROT_EXEC 0x00000004
#define KVME_FLAG_COW 0x00000001
#define KVME_FLAG_NEEDS_COPY 0x00000002
#define KVME_FLAG_NOCOREDUMP 0x00000004
#define KVME_FLAG_SUPER 0x00000008
#define KVME_FLAG_GROWS_UP 0x00000010
#define KVME_FLAG_GROWS_DOWN 0x00000020
#define KVME_FLAG_USER_WIRED 0x00000040
#if defined(__amd64__)
#define KINFO_OVMENTRY_SIZE 1168
#endif
#if defined(__i386__)
#define KINFO_OVMENTRY_SIZE 1128
#endif
struct kinfo_ovmentry {
int kve_structsize; /* Size of kinfo_vmmapentry. */
int kve_type; /* Type of map entry. */
void *kve_start; /* Starting address. */
void *kve_end; /* Finishing address. */
int kve_flags; /* Flags on map entry. */
int kve_resident; /* Number of resident pages. */
int kve_private_resident; /* Number of private pages. */
int kve_protection; /* Protection bitmask. */
int kve_ref_count; /* VM obj ref count. */
int kve_shadow_count; /* VM obj shadow count. */
char kve_path[PATH_MAX]; /* Path to VM obj, if any. */
void *_kve_pspare[8]; /* Space for more stuff. */
off_t kve_offset; /* Mapping offset in object */
uint64_t kve_fileid; /* inode number if vnode */
uint32_t kve_fsid; /* dev_t of vnode location */
int _kve_ispare[3]; /* Space for more stuff. */
};
#if defined(__amd64__) || defined(__i386__)
#define KINFO_VMENTRY_SIZE 1160
#endif
struct kinfo_vmentry {
int kve_structsize; /* Variable size of record. */
int kve_type; /* Type of map entry. */
uint64_t kve_start; /* Starting address. */
uint64_t kve_end; /* Finishing address. */
uint64_t kve_offset; /* Mapping offset in object */
uint64_t kve_vn_fileid; /* inode number if vnode */
uint32_t kve_vn_fsid_freebsd11; /* dev_t of vnode location */
int kve_flags; /* Flags on map entry. */
int kve_resident; /* Number of resident pages. */
int kve_private_resident; /* Number of private pages. */
int kve_protection; /* Protection bitmask. */
int kve_ref_count; /* VM obj ref count. */
int kve_shadow_count; /* VM obj shadow count. */
int kve_vn_type; /* Vnode type. */
uint64_t kve_vn_size; /* File size. */
uint32_t kve_vn_rdev_freebsd11; /* Device id if device. */
uint16_t kve_vn_mode; /* File mode. */
uint16_t kve_status; /* Status flags. */
uint64_t kve_vn_fsid; /* dev_t of vnode location */
uint64_t kve_vn_rdev; /* Device id if device. */
int _kve_ispare[8]; /* Space for more stuff. */
/* Truncated before copyout in sysctl */
char kve_path[PATH_MAX]; /* Path to VM obj, if any. */
};
/*
* The "vm.objects" sysctl provides a list of all VM objects in the system
* via an array of these entries.
*/
struct kinfo_vmobject {
int kvo_structsize; /* Variable size of record. */
int kvo_type; /* Object type: KVME_TYPE_*. */
uint64_t kvo_size; /* Object size in pages. */
uint64_t kvo_vn_fileid; /* inode number if vnode. */
uint32_t kvo_vn_fsid_freebsd11; /* dev_t of vnode location. */
int kvo_ref_count; /* Reference count. */
int kvo_shadow_count; /* Shadow count. */
int kvo_memattr; /* Memory attribute. */
uint64_t kvo_resident; /* Number of resident pages. */
uint64_t kvo_active; /* Number of active pages. */
uint64_t kvo_inactive; /* Number of inactive pages. */
uint64_t kvo_vn_fsid;
uint64_t _kvo_qspare[7];
uint32_t _kvo_ispare[8];
char kvo_path[PATH_MAX]; /* Pathname, if any. */
};
/*
* The KERN_PROC_KSTACK sysctl allows a process to dump the kernel stacks of
* another process as a series of entries. Each stack is represented by a
* series of symbol names and offsets as generated by stack_sbuf_print(9).
*/
#define KKST_MAXLEN 1024
#define KKST_STATE_STACKOK 0 /* Stack is valid. */
#define KKST_STATE_SWAPPED 1 /* Stack swapped out. */
#define KKST_STATE_RUNNING 2 /* Stack ephemeral. */
#if defined(__amd64__) || defined(__i386__)
#define KINFO_KSTACK_SIZE 1096
#endif
struct kinfo_kstack {
lwpid_t kkst_tid; /* ID of thread. */
int kkst_state; /* Validity of stack. */
char kkst_trace[KKST_MAXLEN]; /* String representing stack. */
int _kkst_ispare[16]; /* Space for more stuff. */
};
struct kinfo_sigtramp {
void *ksigtramp_start;
void *ksigtramp_end;
void *ksigtramp_spare[4];
};
#ifdef _KERNEL
/* Flags for kern_proc_out function. */
#define KERN_PROC_NOTHREADS 0x1
#define KERN_PROC_MASK32 0x2
/* Flags for kern_proc_filedesc_out. */
#define KERN_FILEDESC_PACK_KINFO 0x00000001U
/* Flags for kern_proc_vmmap_out. */
#define KERN_VMMAP_PACK_KINFO 0x00000001U
struct sbuf;
/*
* The kern_proc out functions are helper functions to dump process
* miscellaneous kinfo structures to sbuf. The main consumers are KERN_PROC
* sysctls but they may also be used by other kernel subsystems.
*
* The functions manipulate the process locking state and expect the process
* to be locked on enter. On return the process is unlocked.
*/
int kern_proc_filedesc_out(struct proc *p, struct sbuf *sb, ssize_t maxlen,
int flags);
int kern_proc_cwd_out(struct proc *p, struct sbuf *sb, ssize_t maxlen);
int kern_proc_out(struct proc *p, struct sbuf *sb, int flags);
int kern_proc_vmmap_out(struct proc *p, struct sbuf *sb, ssize_t maxlen,
int flags);
int vntype_to_kinfo(int vtype);
void pack_kinfo(struct kinfo_file *kif);
#endif /* !_KERNEL */
#endif