Index: head/sys/conf/kern.post.mk =================================================================== --- head/sys/conf/kern.post.mk +++ head/sys/conf/kern.post.mk @@ -185,13 +185,25 @@ ${CC} ${HACK_EXTRA_FLAGS} -nostdlib hack.c -o hack.pico rm -f hack.c -assym.inc: $S/kern/genassym.sh genassym.o +offset.inc: $S/kern/genoffset.sh genoffset.o + NM='${NM}' NMFLAGS='${NMFLAGS}' sh $S/kern/genoffset.sh genoffset.o > ${.TARGET} + +genoffset.o: $S/kern/genoffset.c + ${CC} -c ${CFLAGS:N-flto:N-fno-common} $S/kern/genoffset.c + +genoffset_test.c: $S/kern/genoffset.c + cp $S/kern/genoffset.c genoffset_test.c + +genoffset_test.o: genoffset_test.c offset.inc + ${CC} -c ${CFLAGS:N-flto:N-fno-common} -DOFFSET_TEST genoffset_test.c + +assym.inc: $S/kern/genassym.sh genassym.o genoffset_test.o NM='${NM}' NMFLAGS='${NMFLAGS}' sh $S/kern/genassym.sh genassym.o > ${.TARGET} -genassym.o: $S/$M/$M/genassym.c +genassym.o: $S/$M/$M/genassym.c offset.inc ${CC} -c ${CFLAGS:N-flto:N-fno-common} $S/$M/$M/genassym.c -${SYSTEM_OBJS} genassym.o vers.o: opt_global.h +${SYSTEM_OBJS} genoffset.o genassym.o vers.o: opt_global.h .if !empty(.MAKE.MODE:Unormal:Mmeta) && empty(.MAKE.MODE:Unormal:Mnofilemon) _meta_filemon= 1 @@ -213,10 +225,10 @@ .endif kernel-depend: .depend -SRCS= assym.inc vnode_if.h ${BEFORE_DEPEND} ${CFILES} \ +SRCS= assym.inc offset.inc vnode_if.h ${BEFORE_DEPEND} ${CFILES} \ ${SYSTEM_CFILES} ${GEN_CFILES} ${SFILES} \ ${MFILES:T:S/.m$/.h/} -DEPENDOBJS+= ${SYSTEM_OBJS} genassym.o +DEPENDOBJS+= ${SYSTEM_OBJS} genassym.o genoffset.o DEPENDFILES= ${DEPENDOBJS:O:u:C/^/.depend./} .if ${MAKE_VERSION} < 20160220 DEPEND_MP?= -MP Index: head/sys/conf/kern.pre.mk =================================================================== --- head/sys/conf/kern.pre.mk +++ head/sys/conf/kern.pre.mk @@ -195,7 +195,7 @@ OFED_C_NOIMP= ${CC} -c -o ${.TARGET} ${OFEDCFLAGS} ${WERROR} ${PROF} OFED_C= ${OFED_C_NOIMP} ${.IMPSRC} -GEN_CFILES= $S/$M/$M/genassym.c ${MFILES:T:S/.m$/.c/} +GEN_CFILES= $S/$M/$M/genassym.c $S/kern/genoffset.c ${MFILES:T:S/.m$/.c/} SYSTEM_CFILES= config.c env.c hints.c vnode_if.c SYSTEM_DEP= Makefile ${SYSTEM_OBJS} SYSTEM_OBJS= locore.o ${MDOBJS} ${OBJS} Index: head/sys/kern/genoffset.c =================================================================== --- head/sys/kern/genoffset.c +++ head/sys/kern/genoffset.c @@ -0,0 +1,44 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2018, Matthew Macy + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef OFFSET_TEST +#define GENOFFSET +#endif +#include +__FBSDID("$FreeBSD$"); +#include +#include +#include + +OFFSYM(td_pre_epoch_prio, thread, u_char); +OFFSYM(td_priority, thread, u_char); +OFFSYM(td_epochnest, thread, u_char); +OFFSYM(td_critnest, thread, u_int); +OFFSYM(td_pinned, thread, int); +OFFSYM(td_owepreempt, thread, u_char); Index: head/sys/kern/genoffset.sh =================================================================== --- head/sys/kern/genoffset.sh +++ head/sys/kern/genoffset.sh @@ -0,0 +1,141 @@ +#!/bin/sh + +# SPDX-License-Identifier: BSD-2-Clause-FreeBSD +# +# Copyright (c) 2000, Bruce Evans +# Copyright (c) 2018, Jeff Roberson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD$ + +usage() +{ + echo "usage: genoffset [-o outfile] objfile" + exit 1 +} + + +work() +{ + echo "#ifndef _OFFSET_INC_" + echo "#define _OFFSET_INC_" + echo "#if !defined(GENOFFSET) && !defined(KLD_MODULE)" + ${NM:='nm'} ${NMFLAGS} "$1" | ${AWK:='awk'} ' + / C .*_datatype_*/ { + type = substr($3, match($3, "_datatype_") + length("_datatype_")) + } + / C .*_parenttype_*/ { + parent = substr($3, match($3, "_parenttype_") + length("_parenttype_")) + } + / C .*sign$/ { + sign = substr($1, length($1) - 3, 4) + sub("^0*", "", sign) + if (sign != "") + sign = "-" + } + / C .*w0$/ { + w0 = substr($1, length($1) - 3, 4) + } + / C .*w1$/ { + w1 = substr($1, length($1) - 3, 4) + } + / C .*w2$/ { + w2 = substr($1, length($1) - 3, 4) + } + / C .*w3$/ { + w3 = substr($1, length($1) - 3, 4) + w = w3 w2 w1 w0 + sub("^0*", "", w) + if (w == "") + w = "0" + hex = "" + if (w != "0") + hex = "0x" + sub("w3$", "", $3) + member = tolower($3) + # This still has minor problems representing INT_MIN, etc. + # E.g., + # with 32-bit 2''s complement ints, this prints -0x80000000, + # which has the wrong type (unsigned int). + offset = sprintf("%s%s%s", sign, hex, w) + + structures[parent] = sprintf("%s%s %s %s\n", + structures[parent], offset, type, member) + } + END { + for (struct in structures) { + printf("struct %s_lite {\n", struct); + n = split(structures[struct], members, "\n") + for (i = 1; i < n; i++) { + for (j = i + 1; j < n; j++) { + split(members[i], ivar, " ") + split(members[j], jvar, " ") + if (jvar[1] < ivar[1]) { + tmp = members[i] + members[i] = members[j] + members[j] = tmp + } + } + } + off = "0" + for (i = 1; i < n; i++) { + split(members[i], m, " ") + printf "\tu_char\tpad_%s[%s - %s];\n", m[3], m[1], off + printf "\t%s\t%s;\n", m[2], m[3] + off = sprintf("(%s + sizeof(%s))", m[1], m[2]) + } + printf("};\n"); + } + } + ' + + echo "#endif" + echo "#endif" +} + + +# +#MAIN PROGGRAM +# +use_outfile="no" +while getopts "o:" option +do + case "$option" in + o) outfile="$OPTARG" + use_outfile="yes";; + *) usage;; + esac +done +shift $(($OPTIND - 1)) +case $# in +1) ;; +*) usage;; +esac + +if [ "$use_outfile" = "yes" ] +then + work $1 3>"$outfile" >&3 3>&- +else + work $1 +fi + Index: head/sys/kern/kern_switch.c =================================================================== --- head/sys/kern/kern_switch.c +++ head/sys/kern/kern_switch.c @@ -199,17 +199,17 @@ * the function call itself, for most cases. */ void -critical_enter(void) +critical_enter_KBI(void) { - struct thread *td; - - td = curthread; - td->td_critnest++; +#ifdef KTR + struct thread *td = curthread; +#endif + critical_enter(); CTR4(KTR_CRITICAL, "critical_enter by thread %p (%ld, %s) to %d", td, (long)td->td_proc->p_pid, td->td_name, td->td_critnest); } -static void __noinline +void __noinline critical_exit_preempt(void) { struct thread *td; @@ -245,17 +245,12 @@ } void -critical_exit(void) +critical_exit_KBI(void) { - struct thread *td; - - td = curthread; - KASSERT(td->td_critnest != 0, - ("critical_exit: td_critnest == 0")); - td->td_critnest--; - __compiler_membar(); - if (__predict_false(td->td_owepreempt)) - critical_exit_preempt(); +#ifdef KTR + struct thread *td = curthread; +#endif + critical_exit(); CTR4(KTR_CRITICAL, "critical_exit by thread %p (%ld, %s) to %d", td, (long)td->td_proc->p_pid, td->td_name, td->td_critnest); } Index: head/sys/sys/assym.h =================================================================== --- head/sys/sys/assym.h +++ head/sys/sys/assym.h @@ -43,4 +43,19 @@ char name ## w2[((ASSYM_ABS(value) & 0xFFFF00000000ULL) >> 32) + ASSYM_BIAS]; \ char name ## w3[((ASSYM_ABS(value) & 0xFFFF000000000000ULL) >> 48) + ASSYM_BIAS] + +/* char name ## _datatype_ ## STRINGIFY(typeof(((struct parenttype *)(0x0))-> name)) [1]; */ +#ifdef OFFSET_TEST +#define OFFSET_CTASSERT CTASSERT +#else +#define OFFSET_CTASSERT(...) +#endif + +#define OFFSYM(name, parenttype, datatype) \ +ASSYM(name, offsetof(struct parenttype, name)); \ +char name ## _datatype_ ## datatype [1]; \ +char name ## _parenttype_ ## parenttype [1]; \ +CTASSERT(__builtin_types_compatible_p(__typeof(((struct parenttype *)(0x0))-> name), datatype)); \ +OFFSET_CTASSERT(offsetof(struct parenttype, name) == offsetof(struct parenttype ## _lite, name)) + #endif /* !_SYS_ASSYM_H_ */ Index: head/sys/sys/kpilite.h =================================================================== --- head/sys/sys/kpilite.h +++ head/sys/sys/kpilite.h @@ -0,0 +1,55 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2018, Matthew Macy + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SYS_KPILITE_H_ +#define _SYS_KPILITE_H_ +#if !defined(GENOFFSET) && !defined(KLD_MODULE) && defined(_KERNEL) +#include "offset.inc" + +static __inline void +sched_pin_lite(struct thread_lite *td) +{ + + KASSERT((struct thread *)td == curthread, ("sched_pin called on non curthread")); + td->td_pinned++; + __compiler_membar(); +} + +static __inline void +sched_unpin_lite(struct thread_lite *td) +{ + + KASSERT((struct thread *)td == curthread, ("sched_unpin called on non curthread")); + KASSERT(td->td_pinned > 0, ("sched_unpin called on non pinned thread")); + __compiler_membar(); + td->td_pinned--; + __compiler_membar(); +} +#endif +#endif Index: head/sys/sys/systm.h =================================================================== --- head/sys/sys/systm.h +++ head/sys/sys/systm.h @@ -107,6 +107,12 @@ #define CTASSERT(x) _Static_assert(x, "compile-time assertion failed") #endif +#if defined(_KERNEL) +#include /* MAXCPU */ +#include /* curthread */ +#include +#endif + /* * Assert that a pointer can be loaded from memory atomically. * @@ -214,12 +220,44 @@ void cpu_boot(int); void cpu_flush_dcache(void *, size_t); void cpu_rootconf(void); -void critical_enter(void); -void critical_exit(void); +void critical_enter_KBI(void); +void critical_exit_KBI(void); +void critical_exit_preempt(void); void init_param1(void); void init_param2(long physpages); void init_static_kenv(char *, size_t); void tablefull(const char *); + +#if defined(KLD_MODULE) || defined(KTR_CRITICAL) || !defined(_KERNEL) || defined(GENOFFSET) +#define critical_enter() critical_enter_KBI() +#define critical_exit() critical_exit_KBI() +#else +static __inline void +critical_enter(void) +{ + struct thread_lite *td; + + td = (struct thread_lite *)curthread; + td->td_critnest++; +} + +static __inline void +critical_exit(void) +{ + struct thread_lite *td; + + td = (struct thread_lite *)curthread; + KASSERT(td->td_critnest != 0, + ("critical_exit: td_critnest == 0")); + td->td_critnest--; + __compiler_membar(); + if (__predict_false(td->td_owepreempt)) + critical_exit_preempt(); + +} +#endif + + #ifdef EARLY_PRINTF typedef void early_putc_t(int ch); extern early_putc_t *early_putc;